

<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  <meta name="Description" content="scikit-learn: machine learning in Python">

  
  <title>3.3. Metrics and scoring: quantifying the quality of predictions &mdash; scikit-learn 0.22 documentation</title>
  
  <link rel="canonical" href="http://scikit-learn.org/stable/modules/model_evaluation.html" />

  
  <link rel="shortcut icon" href="../_static/favicon.ico"/>
  

  <link rel="stylesheet" href="../_static/css/vendor/bootstrap.min.css" type="text/css" />
  <link rel="stylesheet" href="../_static/gallery.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
<script id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
<script src="../_static/jquery.js"></script> 
</head>
<body>
<nav id="navbar" class="sk-docs-navbar navbar navbar-expand-md navbar-light bg-light py-0">
  <div class="container-fluid sk-docs-container px-0">
      <a class="navbar-brand py-0" href="../index.html">
        <img
          class="sk-brand-img"
          src="../_static/scikit-learn-logo-small.png"
          alt="logo"/>
      </a>
    <button
      id="sk-navbar-toggler"
      class="navbar-toggler"
      type="button"
      data-toggle="collapse"
      data-target="#navbarSupportedContent"
      aria-controls="navbarSupportedContent"
      aria-expanded="false"
      aria-label="Toggle navigation"
    >
      <span class="navbar-toggler-icon"></span>
    </button>

    <div class="sk-navbar-collapse collapse navbar-collapse" id="navbarSupportedContent">
      <ul class="navbar-nav mr-auto">
        <li class="nav-item">
          <a class="sk-nav-link nav-link" href="../install.html">Install</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link" href="../user_guide.html">User Guide</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link" href="classes.html">API</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link" href="../auto_examples/index.html">Examples</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../getting_started.html">Getting Started</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../tutorial/index.html">Tutorial</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../glossary.html">Glossary</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../developers/index.html">Development</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../faq.html">FAQ</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../related_projects.html">Related packages</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../roadmap.html">Roadmap</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="../about.html">About us</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="https://github.com/scikit-learn/scikit-learn">GitHub</a>
        </li>
        <li class="nav-item">
          <a class="sk-nav-link nav-link nav-more-item-mobile-items" href="https://scikit-learn.org/dev/versions.html">Other Versions</a>
        </li>
        <li class="nav-item dropdown nav-more-item-dropdown">
          <a class="sk-nav-link nav-link dropdown-toggle" href="#" id="navbarDropdown" role="button" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">More</a>
          <div class="dropdown-menu" aria-labelledby="navbarDropdown">
              <a class="sk-nav-dropdown-item dropdown-item" href="../getting_started.html">Getting Started</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../tutorial/index.html">Tutorial</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../glossary.html">Glossary</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../developers/index.html">Development</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../faq.html">FAQ</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../related_projects.html">Related packages</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../roadmap.html">Roadmap</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="../about.html">About us</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="https://github.com/scikit-learn/scikit-learn">GitHub</a>
              <a class="sk-nav-dropdown-item dropdown-item" href="https://scikit-learn.org/dev/versions.html">Other Versions</a>
          </div>
        </li>
      </ul>
      <div id="searchbox" role="search">
          <div class="searchformwrapper">
          <form class="search" action="../search.html" method="get">
            <input class="sk-search-text-input" type="text" name="q" aria-labelledby="searchlabel" />
            <input class="sk-search-text-btn" type="submit" value="Go" />
          </form>
          </div>
      </div>
    </div>
  </div>
</nav>
<div class="d-flex" id="sk-doc-wrapper">
    <input type="checkbox" name="sk-toggle-checkbox" id="sk-toggle-checkbox">
    <label id="sk-sidemenu-toggle" class="sk-btn-toggle-toc btn sk-btn-primary" for="sk-toggle-checkbox">Toggle Menu</label>
    <div id="sk-sidebar-wrapper" class="border-right">
      <div class="sk-sidebar-toc-wrapper">
        <div class="sk-sidebar-toc-logo">
          <a href="../index.html">
            <img
              class="sk-brand-img"
              src="../_static/scikit-learn-logo-small.png"
              alt="logo"/>
          </a>
        </div>
        <div class="btn-group w-100 mb-2" role="group" aria-label="rellinks">
            <a href="generated/sklearn.ensemble.GradientBoostingRegressor.html" role="button" class="btn sk-btn-rellink py-1" sk-rellink-tooltip="3.2.4.3.6. sklearn.ensemble.GradientBoostingRegressor">Prev</a><a href="../model_selection.html" role="button" class="btn sk-btn-rellink py-1" sk-rellink-tooltip="3. Model selection and evaluation">Up</a>
            <a href="model_persistence.html" role="button" class="btn sk-btn-rellink py-1" sk-rellink-tooltip="3.4. Model persistence">Next</a>
        </div>
        <div class="alert alert-danger p-1 mb-2" role="alert">
          <p class="text-center mb-0">
          <strong>scikit-learn 0.22</strong><br/>
          <a href="http://scikit-learn.org/dev/versions.html">Other versions</a>
          </p>
        </div>
        <div class="alert alert-warning p-1 mb-2" role="alert">
          <p class="text-center mb-0">
            Please <a class="font-weight-bold" href="../about.html#citing-scikit-learn"><string>cite us</string></a> if you use the software.
          </p>
        </div>
          <div class="sk-sidebar-toc">
            <ul>
<li><a class="reference internal" href="#">3.3. Metrics and scoring: quantifying the quality of predictions</a><ul>
<li><a class="reference internal" href="#the-scoring-parameter-defining-model-evaluation-rules">3.3.1. The <code class="docutils literal notranslate"><span class="pre">scoring</span></code> parameter: defining model evaluation rules</a><ul>
<li><a class="reference internal" href="#common-cases-predefined-values">3.3.1.1. Common cases: predefined values</a></li>
<li><a class="reference internal" href="#defining-your-scoring-strategy-from-metric-functions">3.3.1.2. Defining your scoring strategy from metric functions</a></li>
<li><a class="reference internal" href="#implementing-your-own-scoring-object">3.3.1.3. Implementing your own scoring object</a></li>
<li><a class="reference internal" href="#using-multiple-metric-evaluation">3.3.1.4. Using multiple metric evaluation</a></li>
</ul>
</li>
<li><a class="reference internal" href="#classification-metrics">3.3.2. Classification metrics</a><ul>
<li><a class="reference internal" href="#from-binary-to-multiclass-and-multilabel">3.3.2.1. From binary to multiclass and multilabel</a></li>
<li><a class="reference internal" href="#accuracy-score">3.3.2.2. Accuracy score</a></li>
<li><a class="reference internal" href="#balanced-accuracy-score">3.3.2.3. Balanced accuracy score</a></li>
<li><a class="reference internal" href="#cohen-s-kappa">3.3.2.4. Cohen’s kappa</a></li>
<li><a class="reference internal" href="#confusion-matrix">3.3.2.5. Confusion matrix</a></li>
<li><a class="reference internal" href="#classification-report">3.3.2.6. Classification report</a></li>
<li><a class="reference internal" href="#hamming-loss">3.3.2.7. Hamming loss</a></li>
<li><a class="reference internal" href="#precision-recall-and-f-measures">3.3.2.8. Precision, recall and F-measures</a><ul>
<li><a class="reference internal" href="#binary-classification">3.3.2.8.1. Binary classification</a></li>
<li><a class="reference internal" href="#multiclass-and-multilabel-classification">3.3.2.8.2. Multiclass and multilabel classification</a></li>
</ul>
</li>
<li><a class="reference internal" href="#jaccard-similarity-coefficient-score">3.3.2.9. Jaccard similarity coefficient score</a></li>
<li><a class="reference internal" href="#hinge-loss">3.3.2.10. Hinge loss</a></li>
<li><a class="reference internal" href="#log-loss">3.3.2.11. Log loss</a></li>
<li><a class="reference internal" href="#matthews-correlation-coefficient">3.3.2.12. Matthews correlation coefficient</a></li>
<li><a class="reference internal" href="#multi-label-confusion-matrix">3.3.2.13. Multi-label confusion matrix</a></li>
<li><a class="reference internal" href="#receiver-operating-characteristic-roc">3.3.2.14. Receiver operating characteristic (ROC)</a></li>
<li><a class="reference internal" href="#zero-one-loss">3.3.2.15. Zero one loss</a></li>
<li><a class="reference internal" href="#brier-score-loss">3.3.2.16. Brier score loss</a></li>
</ul>
</li>
<li><a class="reference internal" href="#multilabel-ranking-metrics">3.3.3. Multilabel ranking metrics</a><ul>
<li><a class="reference internal" href="#coverage-error">3.3.3.1. Coverage error</a></li>
<li><a class="reference internal" href="#label-ranking-average-precision">3.3.3.2. Label ranking average precision</a></li>
<li><a class="reference internal" href="#ranking-loss">3.3.3.3. Ranking loss</a></li>
<li><a class="reference internal" href="#normalized-discounted-cumulative-gain">3.3.3.4. Normalized Discounted Cumulative Gain</a></li>
</ul>
</li>
<li><a class="reference internal" href="#regression-metrics">3.3.4. Regression metrics</a><ul>
<li><a class="reference internal" href="#explained-variance-score">3.3.4.1. Explained variance score</a></li>
<li><a class="reference internal" href="#max-error">3.3.4.2. Max error</a></li>
<li><a class="reference internal" href="#mean-absolute-error">3.3.4.3. Mean absolute error</a></li>
<li><a class="reference internal" href="#mean-squared-error">3.3.4.4. Mean squared error</a></li>
<li><a class="reference internal" href="#mean-squared-logarithmic-error">3.3.4.5. Mean squared logarithmic error</a></li>
<li><a class="reference internal" href="#median-absolute-error">3.3.4.6. Median absolute error</a></li>
<li><a class="reference internal" href="#r2-score-the-coefficient-of-determination">3.3.4.7. R² score, the coefficient of determination</a></li>
<li><a class="reference internal" href="#mean-poisson-gamma-and-tweedie-deviances">3.3.4.8. Mean Poisson, Gamma, and Tweedie deviances</a></li>
</ul>
</li>
<li><a class="reference internal" href="#clustering-metrics">3.3.5. Clustering metrics</a></li>
<li><a class="reference internal" href="#dummy-estimators">3.3.6. Dummy estimators</a></li>
</ul>
</li>
</ul>

          </div>
      </div>
    </div>
    <div id="sk-page-content-wrapper">
      <div class="sk-page-content container-fluid body px-md-3" role="main">
        
  <div class="section" id="metrics-and-scoring-quantifying-the-quality-of-predictions">
<span id="model-evaluation"></span><h1>3.3. Metrics and scoring: quantifying the quality of predictions<a class="headerlink" href="#metrics-and-scoring-quantifying-the-quality-of-predictions" title="Permalink to this headline">¶</a></h1>
<p>There are 3 different APIs for evaluating the quality of a model’s
predictions:</p>
<ul class="simple">
<li><p><strong>Estimator score method</strong>: Estimators have a <code class="docutils literal notranslate"><span class="pre">score</span></code> method providing a
default evaluation criterion for the problem they are designed to solve.
This is not discussed on this page, but in each estimator’s documentation.</p></li>
<li><p><strong>Scoring parameter</strong>: Model-evaluation tools using
<a class="reference internal" href="cross_validation.html#cross-validation"><span class="std std-ref">cross-validation</span></a> (such as
<a class="reference internal" href="generated/sklearn.model_selection.cross_val_score.html#sklearn.model_selection.cross_val_score" title="sklearn.model_selection.cross_val_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">model_selection.cross_val_score</span></code></a> and
<a class="reference internal" href="generated/sklearn.model_selection.GridSearchCV.html#sklearn.model_selection.GridSearchCV" title="sklearn.model_selection.GridSearchCV"><code class="xref py py-class docutils literal notranslate"><span class="pre">model_selection.GridSearchCV</span></code></a>) rely on an internal <em>scoring</em> strategy.
This is discussed in the section <a class="reference internal" href="#scoring-parameter"><span class="std std-ref">The scoring parameter: defining model evaluation rules</span></a>.</p></li>
<li><p><strong>Metric functions</strong>: The <code class="xref py py-mod docutils literal notranslate"><span class="pre">metrics</span></code> module implements functions
assessing prediction error for specific purposes. These metrics are detailed
in sections on <a class="reference internal" href="#classification-metrics"><span class="std std-ref">Classification metrics</span></a>,
<a class="reference internal" href="#multilabel-ranking-metrics"><span class="std std-ref">Multilabel ranking metrics</span></a>, <a class="reference internal" href="#regression-metrics"><span class="std std-ref">Regression metrics</span></a> and
<a class="reference internal" href="#clustering-metrics"><span class="std std-ref">Clustering metrics</span></a>.</p></li>
</ul>
<p>Finally, <a class="reference internal" href="#dummy-estimators"><span class="std std-ref">Dummy estimators</span></a> are useful to get a baseline
value of those metrics for random predictions.</p>
<div class="admonition seealso">
<p class="admonition-title">See also</p>
<p>For “pairwise” metrics, between <em>samples</em> and not estimators or
predictions, see the <a class="reference internal" href="metrics.html#metrics"><span class="std std-ref">Pairwise metrics, Affinities and Kernels</span></a> section.</p>
</div>
<div class="section" id="the-scoring-parameter-defining-model-evaluation-rules">
<span id="scoring-parameter"></span><h2>3.3.1. The <code class="docutils literal notranslate"><span class="pre">scoring</span></code> parameter: defining model evaluation rules<a class="headerlink" href="#the-scoring-parameter-defining-model-evaluation-rules" title="Permalink to this headline">¶</a></h2>
<p>Model selection and evaluation using tools, such as
<a class="reference internal" href="generated/sklearn.model_selection.GridSearchCV.html#sklearn.model_selection.GridSearchCV" title="sklearn.model_selection.GridSearchCV"><code class="xref py py-class docutils literal notranslate"><span class="pre">model_selection.GridSearchCV</span></code></a> and
<a class="reference internal" href="generated/sklearn.model_selection.cross_val_score.html#sklearn.model_selection.cross_val_score" title="sklearn.model_selection.cross_val_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">model_selection.cross_val_score</span></code></a>, take a <code class="docutils literal notranslate"><span class="pre">scoring</span></code> parameter that
controls what metric they apply to the estimators evaluated.</p>
<div class="section" id="common-cases-predefined-values">
<h3>3.3.1.1. Common cases: predefined values<a class="headerlink" href="#common-cases-predefined-values" title="Permalink to this headline">¶</a></h3>
<p>For the most common use cases, you can designate a scorer object with the
<code class="docutils literal notranslate"><span class="pre">scoring</span></code> parameter; the table below shows all possible values.
All scorer objects follow the convention that <strong>higher return values are better
than lower return values</strong>.  Thus metrics which measure the distance between
the model and the data, like <a class="reference internal" href="generated/sklearn.metrics.mean_squared_error.html#sklearn.metrics.mean_squared_error" title="sklearn.metrics.mean_squared_error"><code class="xref py py-func docutils literal notranslate"><span class="pre">metrics.mean_squared_error</span></code></a>, are
available as neg_mean_squared_error which return the negated value
of the metric.</p>
<table class="docutils align-default">
<colgroup>
<col style="width: 28%" />
<col style="width: 41%" />
<col style="width: 31%" />
</colgroup>
<thead>
<tr class="row-odd"><th class="head"><p>Scoring</p></th>
<th class="head"><p>Function</p></th>
<th class="head"><p>Comment</p></th>
</tr>
</thead>
<tbody>
<tr class="row-even"><td><p><strong>Classification</strong></p></td>
<td></td>
<td></td>
</tr>
<tr class="row-odd"><td><p>‘accuracy’</p></td>
<td><p><a class="reference internal" href="generated/sklearn.metrics.accuracy_score.html#sklearn.metrics.accuracy_score" title="sklearn.metrics.accuracy_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">metrics.accuracy_score</span></code></a></p></td>
<td></td>
</tr>
<tr class="row-even"><td><p>‘balanced_accuracy’</p></td>
<td><p><a class="reference internal" href="generated/sklearn.metrics.balanced_accuracy_score.html#sklearn.metrics.balanced_accuracy_score" title="sklearn.metrics.balanced_accuracy_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">metrics.balanced_accuracy_score</span></code></a></p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p>‘average_precision’</p></td>
<td><p><a class="reference internal" href="generated/sklearn.metrics.average_precision_score.html#sklearn.metrics.average_precision_score" title="sklearn.metrics.average_precision_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">metrics.average_precision_score</span></code></a></p></td>
<td></td>
</tr>
<tr class="row-even"><td><p>‘neg_brier_score’</p></td>
<td><p><a class="reference internal" href="generated/sklearn.metrics.brier_score_loss.html#sklearn.metrics.brier_score_loss" title="sklearn.metrics.brier_score_loss"><code class="xref py py-func docutils literal notranslate"><span class="pre">metrics.brier_score_loss</span></code></a></p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p>‘f1’</p></td>
<td><p><a class="reference internal" href="generated/sklearn.metrics.f1_score.html#sklearn.metrics.f1_score" title="sklearn.metrics.f1_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">metrics.f1_score</span></code></a></p></td>
<td><p>for binary targets</p></td>
</tr>
<tr class="row-even"><td><p>‘f1_micro’</p></td>
<td><p><a class="reference internal" href="generated/sklearn.metrics.f1_score.html#sklearn.metrics.f1_score" title="sklearn.metrics.f1_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">metrics.f1_score</span></code></a></p></td>
<td><p>micro-averaged</p></td>
</tr>
<tr class="row-odd"><td><p>‘f1_macro’</p></td>
<td><p><a class="reference internal" href="generated/sklearn.metrics.f1_score.html#sklearn.metrics.f1_score" title="sklearn.metrics.f1_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">metrics.f1_score</span></code></a></p></td>
<td><p>macro-averaged</p></td>
</tr>
<tr class="row-even"><td><p>‘f1_weighted’</p></td>
<td><p><a class="reference internal" href="generated/sklearn.metrics.f1_score.html#sklearn.metrics.f1_score" title="sklearn.metrics.f1_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">metrics.f1_score</span></code></a></p></td>
<td><p>weighted average</p></td>
</tr>
<tr class="row-odd"><td><p>‘f1_samples’</p></td>
<td><p><a class="reference internal" href="generated/sklearn.metrics.f1_score.html#sklearn.metrics.f1_score" title="sklearn.metrics.f1_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">metrics.f1_score</span></code></a></p></td>
<td><p>by multilabel sample</p></td>
</tr>
<tr class="row-even"><td><p>‘neg_log_loss’</p></td>
<td><p><a class="reference internal" href="generated/sklearn.metrics.log_loss.html#sklearn.metrics.log_loss" title="sklearn.metrics.log_loss"><code class="xref py py-func docutils literal notranslate"><span class="pre">metrics.log_loss</span></code></a></p></td>
<td><p>requires <code class="docutils literal notranslate"><span class="pre">predict_proba</span></code> support</p></td>
</tr>
<tr class="row-odd"><td><p>‘precision’ etc.</p></td>
<td><p><a class="reference internal" href="generated/sklearn.metrics.precision_score.html#sklearn.metrics.precision_score" title="sklearn.metrics.precision_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">metrics.precision_score</span></code></a></p></td>
<td><p>suffixes apply as with ‘f1’</p></td>
</tr>
<tr class="row-even"><td><p>‘recall’ etc.</p></td>
<td><p><a class="reference internal" href="generated/sklearn.metrics.recall_score.html#sklearn.metrics.recall_score" title="sklearn.metrics.recall_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">metrics.recall_score</span></code></a></p></td>
<td><p>suffixes apply as with ‘f1’</p></td>
</tr>
<tr class="row-odd"><td><p>‘jaccard’ etc.</p></td>
<td><p><a class="reference internal" href="generated/sklearn.metrics.jaccard_score.html#sklearn.metrics.jaccard_score" title="sklearn.metrics.jaccard_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">metrics.jaccard_score</span></code></a></p></td>
<td><p>suffixes apply as with ‘f1’</p></td>
</tr>
<tr class="row-even"><td><p>‘roc_auc’</p></td>
<td><p><a class="reference internal" href="generated/sklearn.metrics.roc_auc_score.html#sklearn.metrics.roc_auc_score" title="sklearn.metrics.roc_auc_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">metrics.roc_auc_score</span></code></a></p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p>‘roc_auc_ovr’</p></td>
<td><p><a class="reference internal" href="generated/sklearn.metrics.roc_auc_score.html#sklearn.metrics.roc_auc_score" title="sklearn.metrics.roc_auc_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">metrics.roc_auc_score</span></code></a></p></td>
<td></td>
</tr>
<tr class="row-even"><td><p>‘roc_auc_ovo’</p></td>
<td><p><a class="reference internal" href="generated/sklearn.metrics.roc_auc_score.html#sklearn.metrics.roc_auc_score" title="sklearn.metrics.roc_auc_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">metrics.roc_auc_score</span></code></a></p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p>‘roc_auc_ovr_weighted’</p></td>
<td><p><a class="reference internal" href="generated/sklearn.metrics.roc_auc_score.html#sklearn.metrics.roc_auc_score" title="sklearn.metrics.roc_auc_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">metrics.roc_auc_score</span></code></a></p></td>
<td></td>
</tr>
<tr class="row-even"><td><p>‘roc_auc_ovo_weighted’</p></td>
<td><p><a class="reference internal" href="generated/sklearn.metrics.roc_auc_score.html#sklearn.metrics.roc_auc_score" title="sklearn.metrics.roc_auc_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">metrics.roc_auc_score</span></code></a></p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p><strong>Clustering</strong></p></td>
<td></td>
<td></td>
</tr>
<tr class="row-even"><td><p>‘adjusted_mutual_info_score’</p></td>
<td><p><a class="reference internal" href="generated/sklearn.metrics.adjusted_mutual_info_score.html#sklearn.metrics.adjusted_mutual_info_score" title="sklearn.metrics.adjusted_mutual_info_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">metrics.adjusted_mutual_info_score</span></code></a></p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p>‘adjusted_rand_score’</p></td>
<td><p><a class="reference internal" href="generated/sklearn.metrics.adjusted_rand_score.html#sklearn.metrics.adjusted_rand_score" title="sklearn.metrics.adjusted_rand_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">metrics.adjusted_rand_score</span></code></a></p></td>
<td></td>
</tr>
<tr class="row-even"><td><p>‘completeness_score’</p></td>
<td><p><a class="reference internal" href="generated/sklearn.metrics.completeness_score.html#sklearn.metrics.completeness_score" title="sklearn.metrics.completeness_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">metrics.completeness_score</span></code></a></p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p>‘fowlkes_mallows_score’</p></td>
<td><p><a class="reference internal" href="generated/sklearn.metrics.fowlkes_mallows_score.html#sklearn.metrics.fowlkes_mallows_score" title="sklearn.metrics.fowlkes_mallows_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">metrics.fowlkes_mallows_score</span></code></a></p></td>
<td></td>
</tr>
<tr class="row-even"><td><p>‘homogeneity_score’</p></td>
<td><p><a class="reference internal" href="generated/sklearn.metrics.homogeneity_score.html#sklearn.metrics.homogeneity_score" title="sklearn.metrics.homogeneity_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">metrics.homogeneity_score</span></code></a></p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p>‘mutual_info_score’</p></td>
<td><p><a class="reference internal" href="generated/sklearn.metrics.mutual_info_score.html#sklearn.metrics.mutual_info_score" title="sklearn.metrics.mutual_info_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">metrics.mutual_info_score</span></code></a></p></td>
<td></td>
</tr>
<tr class="row-even"><td><p>‘normalized_mutual_info_score’</p></td>
<td><p><a class="reference internal" href="generated/sklearn.metrics.normalized_mutual_info_score.html#sklearn.metrics.normalized_mutual_info_score" title="sklearn.metrics.normalized_mutual_info_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">metrics.normalized_mutual_info_score</span></code></a></p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p>‘v_measure_score’</p></td>
<td><p><a class="reference internal" href="generated/sklearn.metrics.v_measure_score.html#sklearn.metrics.v_measure_score" title="sklearn.metrics.v_measure_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">metrics.v_measure_score</span></code></a></p></td>
<td></td>
</tr>
<tr class="row-even"><td><p><strong>Regression</strong></p></td>
<td></td>
<td></td>
</tr>
<tr class="row-odd"><td><p>‘explained_variance’</p></td>
<td><p><a class="reference internal" href="generated/sklearn.metrics.explained_variance_score.html#sklearn.metrics.explained_variance_score" title="sklearn.metrics.explained_variance_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">metrics.explained_variance_score</span></code></a></p></td>
<td></td>
</tr>
<tr class="row-even"><td><p>‘max_error’</p></td>
<td><p><a class="reference internal" href="generated/sklearn.metrics.max_error.html#sklearn.metrics.max_error" title="sklearn.metrics.max_error"><code class="xref py py-func docutils literal notranslate"><span class="pre">metrics.max_error</span></code></a></p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p>‘neg_mean_absolute_error’</p></td>
<td><p><a class="reference internal" href="generated/sklearn.metrics.mean_absolute_error.html#sklearn.metrics.mean_absolute_error" title="sklearn.metrics.mean_absolute_error"><code class="xref py py-func docutils literal notranslate"><span class="pre">metrics.mean_absolute_error</span></code></a></p></td>
<td></td>
</tr>
<tr class="row-even"><td><p>‘neg_mean_squared_error’</p></td>
<td><p><a class="reference internal" href="generated/sklearn.metrics.mean_squared_error.html#sklearn.metrics.mean_squared_error" title="sklearn.metrics.mean_squared_error"><code class="xref py py-func docutils literal notranslate"><span class="pre">metrics.mean_squared_error</span></code></a></p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p>‘neg_root_mean_squared_error’</p></td>
<td><p><a class="reference internal" href="generated/sklearn.metrics.mean_squared_error.html#sklearn.metrics.mean_squared_error" title="sklearn.metrics.mean_squared_error"><code class="xref py py-func docutils literal notranslate"><span class="pre">metrics.mean_squared_error</span></code></a></p></td>
<td></td>
</tr>
<tr class="row-even"><td><p>‘neg_mean_squared_log_error’</p></td>
<td><p><a class="reference internal" href="generated/sklearn.metrics.mean_squared_log_error.html#sklearn.metrics.mean_squared_log_error" title="sklearn.metrics.mean_squared_log_error"><code class="xref py py-func docutils literal notranslate"><span class="pre">metrics.mean_squared_log_error</span></code></a></p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p>‘neg_median_absolute_error’</p></td>
<td><p><a class="reference internal" href="generated/sklearn.metrics.median_absolute_error.html#sklearn.metrics.median_absolute_error" title="sklearn.metrics.median_absolute_error"><code class="xref py py-func docutils literal notranslate"><span class="pre">metrics.median_absolute_error</span></code></a></p></td>
<td></td>
</tr>
<tr class="row-even"><td><p>‘r2’</p></td>
<td><p><a class="reference internal" href="generated/sklearn.metrics.r2_score.html#sklearn.metrics.r2_score" title="sklearn.metrics.r2_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">metrics.r2_score</span></code></a></p></td>
<td></td>
</tr>
<tr class="row-odd"><td><p>‘neg_mean_poisson_deviance’</p></td>
<td><p><a class="reference internal" href="generated/sklearn.metrics.mean_poisson_deviance.html#sklearn.metrics.mean_poisson_deviance" title="sklearn.metrics.mean_poisson_deviance"><code class="xref py py-func docutils literal notranslate"><span class="pre">metrics.mean_poisson_deviance</span></code></a></p></td>
<td></td>
</tr>
<tr class="row-even"><td><p>‘neg_mean_gamma_deviance’</p></td>
<td><p><a class="reference internal" href="generated/sklearn.metrics.mean_gamma_deviance.html#sklearn.metrics.mean_gamma_deviance" title="sklearn.metrics.mean_gamma_deviance"><code class="xref py py-func docutils literal notranslate"><span class="pre">metrics.mean_gamma_deviance</span></code></a></p></td>
<td></td>
</tr>
</tbody>
</table>
<p>Usage examples:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn</span> <span class="kn">import</span> <span class="n">svm</span><span class="p">,</span> <span class="n">datasets</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.model_selection</span> <span class="kn">import</span> <span class="n">cross_val_score</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">X</span><span class="p">,</span> <span class="n">y</span> <span class="o">=</span> <span class="n">datasets</span><span class="o">.</span><span class="n">load_iris</span><span class="p">(</span><span class="n">return_X_y</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">clf</span> <span class="o">=</span> <span class="n">svm</span><span class="o">.</span><span class="n">SVC</span><span class="p">(</span><span class="n">random_state</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">cross_val_score</span><span class="p">(</span><span class="n">clf</span><span class="p">,</span> <span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">cv</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">scoring</span><span class="o">=</span><span class="s1">&#39;recall_macro&#39;</span><span class="p">)</span>
<span class="go">array([0.96..., 0.96..., 0.96..., 0.93..., 1.        ])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">model</span> <span class="o">=</span> <span class="n">svm</span><span class="o">.</span><span class="n">SVC</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">cross_val_score</span><span class="p">(</span><span class="n">model</span><span class="p">,</span> <span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">cv</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">scoring</span><span class="o">=</span><span class="s1">&#39;wrong_choice&#39;</span><span class="p">)</span>
<span class="gt">Traceback (most recent call last):</span>
<span class="gr">ValueError</span>: <span class="n">&#39;wrong_choice&#39; is not a valid scoring value. Use sorted(sklearn.metrics.SCORERS.keys()) to get valid options.</span>
</pre></div>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>The values listed by the ValueError exception correspond to the functions measuring
prediction accuracy described in the following sections.
The scorer objects for those functions are stored in the dictionary
<code class="docutils literal notranslate"><span class="pre">sklearn.metrics.SCORERS</span></code>.</p>
</div>
</div>
<div class="section" id="defining-your-scoring-strategy-from-metric-functions">
<span id="scoring"></span><h3>3.3.1.2. Defining your scoring strategy from metric functions<a class="headerlink" href="#defining-your-scoring-strategy-from-metric-functions" title="Permalink to this headline">¶</a></h3>
<p>The module <a class="reference internal" href="classes.html#module-sklearn.metrics" title="sklearn.metrics"><code class="xref py py-mod docutils literal notranslate"><span class="pre">sklearn.metrics</span></code></a> also exposes a set of simple functions
measuring a prediction error given ground truth and prediction:</p>
<ul class="simple">
<li><p>functions ending with <code class="docutils literal notranslate"><span class="pre">_score</span></code> return a value to
maximize, the higher the better.</p></li>
<li><p>functions ending with <code class="docutils literal notranslate"><span class="pre">_error</span></code> or <code class="docutils literal notranslate"><span class="pre">_loss</span></code> return a
value to minimize, the lower the better.  When converting
into a scorer object using <a class="reference internal" href="generated/sklearn.metrics.make_scorer.html#sklearn.metrics.make_scorer" title="sklearn.metrics.make_scorer"><code class="xref py py-func docutils literal notranslate"><span class="pre">make_scorer</span></code></a>, set
the <code class="docutils literal notranslate"><span class="pre">greater_is_better</span></code> parameter to False (True by default; see the
parameter description below).</p></li>
</ul>
<p>Metrics available for various machine learning tasks are detailed in sections
below.</p>
<p>Many metrics are not given names to be used as <code class="docutils literal notranslate"><span class="pre">scoring</span></code> values,
sometimes because they require additional parameters, such as
<a class="reference internal" href="generated/sklearn.metrics.fbeta_score.html#sklearn.metrics.fbeta_score" title="sklearn.metrics.fbeta_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">fbeta_score</span></code></a>. In such cases, you need to generate an appropriate
scoring object.  The simplest way to generate a callable object for scoring
is by using <a class="reference internal" href="generated/sklearn.metrics.make_scorer.html#sklearn.metrics.make_scorer" title="sklearn.metrics.make_scorer"><code class="xref py py-func docutils literal notranslate"><span class="pre">make_scorer</span></code></a>. That function converts metrics
into callables that can be used for model evaluation.</p>
<p>One typical use case is to wrap an existing metric function from the library
with non-default values for its parameters, such as the <code class="docutils literal notranslate"><span class="pre">beta</span></code> parameter for
the <a class="reference internal" href="generated/sklearn.metrics.fbeta_score.html#sklearn.metrics.fbeta_score" title="sklearn.metrics.fbeta_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">fbeta_score</span></code></a> function:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="kn">import</span> <span class="n">fbeta_score</span><span class="p">,</span> <span class="n">make_scorer</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">ftwo_scorer</span> <span class="o">=</span> <span class="n">make_scorer</span><span class="p">(</span><span class="n">fbeta_score</span><span class="p">,</span> <span class="n">beta</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.model_selection</span> <span class="kn">import</span> <span class="n">GridSearchCV</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.svm</span> <span class="kn">import</span> <span class="n">LinearSVC</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">grid</span> <span class="o">=</span> <span class="n">GridSearchCV</span><span class="p">(</span><span class="n">LinearSVC</span><span class="p">(),</span> <span class="n">param_grid</span><span class="o">=</span><span class="p">{</span><span class="s1">&#39;C&#39;</span><span class="p">:</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">10</span><span class="p">]},</span>
<span class="gp">... </span>                    <span class="n">scoring</span><span class="o">=</span><span class="n">ftwo_scorer</span><span class="p">,</span> <span class="n">cv</span><span class="o">=</span><span class="mi">5</span><span class="p">)</span>
</pre></div>
</div>
<p>The second use case is to build a completely custom scorer object
from a simple python function using <a class="reference internal" href="generated/sklearn.metrics.make_scorer.html#sklearn.metrics.make_scorer" title="sklearn.metrics.make_scorer"><code class="xref py py-func docutils literal notranslate"><span class="pre">make_scorer</span></code></a>, which can
take several parameters:</p>
<ul class="simple">
<li><p>the python function you want to use (<code class="docutils literal notranslate"><span class="pre">my_custom_loss_func</span></code>
in the example below)</p></li>
<li><p>whether the python function returns a score (<code class="docutils literal notranslate"><span class="pre">greater_is_better=True</span></code>,
the default) or a loss (<code class="docutils literal notranslate"><span class="pre">greater_is_better=False</span></code>).  If a loss, the output
of the python function is negated by the scorer object, conforming to
the cross validation convention that scorers return higher values for better models.</p></li>
<li><p>for classification metrics only: whether the python function you provided requires continuous decision
certainties (<code class="docutils literal notranslate"><span class="pre">needs_threshold=True</span></code>).  The default value is
False.</p></li>
<li><p>any additional parameters, such as <code class="docutils literal notranslate"><span class="pre">beta</span></code> or <code class="docutils literal notranslate"><span class="pre">labels</span></code> in <a class="reference internal" href="generated/sklearn.metrics.f1_score.html#sklearn.metrics.f1_score" title="sklearn.metrics.f1_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">f1_score</span></code></a>.</p></li>
</ul>
<p>Here is an example of building custom scorers, and of using the
<code class="docutils literal notranslate"><span class="pre">greater_is_better</span></code> parameter:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">def</span> <span class="nf">my_custom_loss_func</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">):</span>
<span class="gp">... </span>    <span class="n">diff</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">abs</span><span class="p">(</span><span class="n">y_true</span> <span class="o">-</span> <span class="n">y_pred</span><span class="p">)</span><span class="o">.</span><span class="n">max</span><span class="p">()</span>
<span class="gp">... </span>    <span class="k">return</span> <span class="n">np</span><span class="o">.</span><span class="n">log1p</span><span class="p">(</span><span class="n">diff</span><span class="p">)</span>
<span class="gp">...</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># score will negate the return value of my_custom_loss_func,</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># which will be np.log(2), 0.693, given the values for X</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># and y defined below.</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">score</span> <span class="o">=</span> <span class="n">make_scorer</span><span class="p">(</span><span class="n">my_custom_loss_func</span><span class="p">,</span> <span class="n">greater_is_better</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">X</span> <span class="o">=</span> <span class="p">[[</span><span class="mi">1</span><span class="p">],</span> <span class="p">[</span><span class="mi">1</span><span class="p">]]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y</span> <span class="o">=</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.dummy</span> <span class="kn">import</span> <span class="n">DummyClassifier</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">clf</span> <span class="o">=</span> <span class="n">DummyClassifier</span><span class="p">(</span><span class="n">strategy</span><span class="o">=</span><span class="s1">&#39;most_frequent&#39;</span><span class="p">,</span> <span class="n">random_state</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">clf</span> <span class="o">=</span> <span class="n">clf</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">my_custom_loss_func</span><span class="p">(</span><span class="n">clf</span><span class="o">.</span><span class="n">predict</span><span class="p">(</span><span class="n">X</span><span class="p">),</span> <span class="n">y</span><span class="p">)</span>
<span class="go">0.69...</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">score</span><span class="p">(</span><span class="n">clf</span><span class="p">,</span> <span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span>
<span class="go">-0.69...</span>
</pre></div>
</div>
</div>
<div class="section" id="implementing-your-own-scoring-object">
<span id="diy-scoring"></span><h3>3.3.1.3. Implementing your own scoring object<a class="headerlink" href="#implementing-your-own-scoring-object" title="Permalink to this headline">¶</a></h3>
<p>You can generate even more flexible model scorers by constructing your own
scoring object from scratch, without using the <a class="reference internal" href="generated/sklearn.metrics.make_scorer.html#sklearn.metrics.make_scorer" title="sklearn.metrics.make_scorer"><code class="xref py py-func docutils literal notranslate"><span class="pre">make_scorer</span></code></a> factory.
For a callable to be a scorer, it needs to meet the protocol specified by
the following two rules:</p>
<ul class="simple">
<li><p>It can be called with parameters <code class="docutils literal notranslate"><span class="pre">(estimator,</span> <span class="pre">X,</span> <span class="pre">y)</span></code>, where <code class="docutils literal notranslate"><span class="pre">estimator</span></code>
is the model that should be evaluated, <code class="docutils literal notranslate"><span class="pre">X</span></code> is validation data, and <code class="docutils literal notranslate"><span class="pre">y</span></code> is
the ground truth target for <code class="docutils literal notranslate"><span class="pre">X</span></code> (in the supervised case) or <code class="docutils literal notranslate"><span class="pre">None</span></code> (in the
unsupervised case).</p></li>
<li><p>It returns a floating point number that quantifies the
<code class="docutils literal notranslate"><span class="pre">estimator</span></code> prediction quality on <code class="docutils literal notranslate"><span class="pre">X</span></code>, with reference to <code class="docutils literal notranslate"><span class="pre">y</span></code>.
Again, by convention higher numbers are better, so if your scorer
returns loss, that value should be negated.</p></li>
</ul>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p><strong>Using custom scorers in functions where n_jobs &gt; 1</strong></p>
<p>While defining the custom scoring function alongside the calling function
should work out of the box with the default joblib backend (loky),
importing it from another module will be a more robust approach and work
independently of the joblib backend.</p>
<p>For example, to use <code class="docutils literal notranslate"><span class="pre">n_jobs</span></code> greater than 1 in the example below,
<code class="docutils literal notranslate"><span class="pre">custom_scoring_function</span></code> function is saved in a user-created module
(<code class="docutils literal notranslate"><span class="pre">custom_scorer_module.py</span></code>) and imported:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">custom_scorer_module</span> <span class="kn">import</span> <span class="n">custom_scoring_function</span> 
<span class="gp">&gt;&gt;&gt; </span><span class="n">cross_val_score</span><span class="p">(</span><span class="n">model</span><span class="p">,</span>
<span class="gp">... </span> <span class="n">X_train</span><span class="p">,</span>
<span class="gp">... </span> <span class="n">y_train</span><span class="p">,</span>
<span class="gp">... </span> <span class="n">scoring</span><span class="o">=</span><span class="n">make_scorer</span><span class="p">(</span><span class="n">custom_scoring_function</span><span class="p">,</span> <span class="n">greater_is_better</span><span class="o">=</span><span class="kc">False</span><span class="p">),</span>
<span class="gp">... </span> <span class="n">cv</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span>
<span class="gp">... </span> <span class="n">n_jobs</span><span class="o">=-</span><span class="mi">1</span><span class="p">)</span> 
</pre></div>
</div>
</div>
</div>
<div class="section" id="using-multiple-metric-evaluation">
<span id="multimetric-scoring"></span><h3>3.3.1.4. Using multiple metric evaluation<a class="headerlink" href="#using-multiple-metric-evaluation" title="Permalink to this headline">¶</a></h3>
<p>Scikit-learn also permits evaluation of multiple metrics in <code class="docutils literal notranslate"><span class="pre">GridSearchCV</span></code>,
<code class="docutils literal notranslate"><span class="pre">RandomizedSearchCV</span></code> and <code class="docutils literal notranslate"><span class="pre">cross_validate</span></code>.</p>
<p>There are two ways to specify multiple scoring metrics for the <code class="docutils literal notranslate"><span class="pre">scoring</span></code>
parameter:</p>
<ul>
<li><dl>
<dt>As an iterable of string metrics::</dt><dd><div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">scoring</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;accuracy&#39;</span><span class="p">,</span> <span class="s1">&#39;precision&#39;</span><span class="p">]</span>
</pre></div>
</div>
</dd>
</dl>
</li>
<li><dl>
<dt>As a <code class="docutils literal notranslate"><span class="pre">dict</span></code> mapping the scorer name to the scoring function::</dt><dd><div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="kn">import</span> <span class="n">accuracy_score</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="kn">import</span> <span class="n">make_scorer</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">scoring</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;accuracy&#39;</span><span class="p">:</span> <span class="n">make_scorer</span><span class="p">(</span><span class="n">accuracy_score</span><span class="p">),</span>
<span class="gp">... </span>           <span class="s1">&#39;prec&#39;</span><span class="p">:</span> <span class="s1">&#39;precision&#39;</span><span class="p">}</span>
</pre></div>
</div>
</dd>
</dl>
</li>
</ul>
<p>Note that the dict values can either be scorer functions or one of the
predefined metric strings.</p>
<p>Currently only those scorer functions that return a single score can be passed
inside the dict. Scorer functions that return multiple values are not
permitted and will require a wrapper to return a single metric:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.model_selection</span> <span class="kn">import</span> <span class="n">cross_validate</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="kn">import</span> <span class="n">confusion_matrix</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># A sample toy binary classification dataset</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">X</span><span class="p">,</span> <span class="n">y</span> <span class="o">=</span> <span class="n">datasets</span><span class="o">.</span><span class="n">make_classification</span><span class="p">(</span><span class="n">n_classes</span><span class="o">=</span><span class="mi">2</span><span class="p">,</span> <span class="n">random_state</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">svm</span> <span class="o">=</span> <span class="n">LinearSVC</span><span class="p">(</span><span class="n">random_state</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">def</span> <span class="nf">tn</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">):</span> <span class="k">return</span> <span class="n">confusion_matrix</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">)[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">def</span> <span class="nf">fp</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">):</span> <span class="k">return</span> <span class="n">confusion_matrix</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">)[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">def</span> <span class="nf">fn</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">):</span> <span class="k">return</span> <span class="n">confusion_matrix</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">)[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="k">def</span> <span class="nf">tp</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">):</span> <span class="k">return</span> <span class="n">confusion_matrix</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">)[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">scoring</span> <span class="o">=</span> <span class="p">{</span><span class="s1">&#39;tp&#39;</span><span class="p">:</span> <span class="n">make_scorer</span><span class="p">(</span><span class="n">tp</span><span class="p">),</span> <span class="s1">&#39;tn&#39;</span><span class="p">:</span> <span class="n">make_scorer</span><span class="p">(</span><span class="n">tn</span><span class="p">),</span>
<span class="gp">... </span>           <span class="s1">&#39;fp&#39;</span><span class="p">:</span> <span class="n">make_scorer</span><span class="p">(</span><span class="n">fp</span><span class="p">),</span> <span class="s1">&#39;fn&#39;</span><span class="p">:</span> <span class="n">make_scorer</span><span class="p">(</span><span class="n">fn</span><span class="p">)}</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">cv_results</span> <span class="o">=</span> <span class="n">cross_validate</span><span class="p">(</span><span class="n">svm</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">),</span> <span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">cv</span><span class="o">=</span><span class="mi">5</span><span class="p">,</span> <span class="n">scoring</span><span class="o">=</span><span class="n">scoring</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Getting the test set true positive scores</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">print</span><span class="p">(</span><span class="n">cv_results</span><span class="p">[</span><span class="s1">&#39;test_tp&#39;</span><span class="p">])</span>
<span class="go">[10  9  8  7  8]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># Getting the test set false negative scores</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">print</span><span class="p">(</span><span class="n">cv_results</span><span class="p">[</span><span class="s1">&#39;test_fn&#39;</span><span class="p">])</span>
<span class="go">[0 1 2 3 2]</span>
</pre></div>
</div>
</div>
</div>
<div class="section" id="classification-metrics">
<span id="id1"></span><h2>3.3.2. Classification metrics<a class="headerlink" href="#classification-metrics" title="Permalink to this headline">¶</a></h2>
<p>The <a class="reference internal" href="classes.html#module-sklearn.metrics" title="sklearn.metrics"><code class="xref py py-mod docutils literal notranslate"><span class="pre">sklearn.metrics</span></code></a> module implements several loss, score, and utility
functions to measure classification performance.
Some metrics might require probability estimates of the positive class,
confidence values, or binary decisions values.
Most implementations allow each sample to provide a weighted contribution
to the overall score, through the <code class="docutils literal notranslate"><span class="pre">sample_weight</span></code> parameter.</p>
<p>Some of these are restricted to the binary classification case:</p>
<table class="longtable docutils align-default">
<colgroup>
<col style="width: 10%" />
<col style="width: 90%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p><a class="reference internal" href="generated/sklearn.metrics.precision_recall_curve.html#sklearn.metrics.precision_recall_curve" title="sklearn.metrics.precision_recall_curve"><code class="xref py py-obj docutils literal notranslate"><span class="pre">precision_recall_curve</span></code></a>(y_true, probas_pred)</p></td>
<td><p>Compute precision-recall pairs for different probability thresholds</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="generated/sklearn.metrics.roc_curve.html#sklearn.metrics.roc_curve" title="sklearn.metrics.roc_curve"><code class="xref py py-obj docutils literal notranslate"><span class="pre">roc_curve</span></code></a>(y_true, y_score[, pos_label, …])</p></td>
<td><p>Compute Receiver operating characteristic (ROC)</p></td>
</tr>
</tbody>
</table>
<p>Others also work in the multiclass case:</p>
<table class="longtable docutils align-default">
<colgroup>
<col style="width: 10%" />
<col style="width: 90%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p><a class="reference internal" href="generated/sklearn.metrics.balanced_accuracy_score.html#sklearn.metrics.balanced_accuracy_score" title="sklearn.metrics.balanced_accuracy_score"><code class="xref py py-obj docutils literal notranslate"><span class="pre">balanced_accuracy_score</span></code></a>(y_true, y_pred[, …])</p></td>
<td><p>Compute the balanced accuracy</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="generated/sklearn.metrics.cohen_kappa_score.html#sklearn.metrics.cohen_kappa_score" title="sklearn.metrics.cohen_kappa_score"><code class="xref py py-obj docutils literal notranslate"><span class="pre">cohen_kappa_score</span></code></a>(y1, y2[, labels, weights, …])</p></td>
<td><p>Cohen’s kappa: a statistic that measures inter-annotator agreement.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="generated/sklearn.metrics.confusion_matrix.html#sklearn.metrics.confusion_matrix" title="sklearn.metrics.confusion_matrix"><code class="xref py py-obj docutils literal notranslate"><span class="pre">confusion_matrix</span></code></a>(y_true, y_pred[, labels, …])</p></td>
<td><p>Compute confusion matrix to evaluate the accuracy of a classification.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="generated/sklearn.metrics.hinge_loss.html#sklearn.metrics.hinge_loss" title="sklearn.metrics.hinge_loss"><code class="xref py py-obj docutils literal notranslate"><span class="pre">hinge_loss</span></code></a>(y_true, pred_decision[, labels, …])</p></td>
<td><p>Average hinge loss (non-regularized)</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="generated/sklearn.metrics.matthews_corrcoef.html#sklearn.metrics.matthews_corrcoef" title="sklearn.metrics.matthews_corrcoef"><code class="xref py py-obj docutils literal notranslate"><span class="pre">matthews_corrcoef</span></code></a>(y_true, y_pred[, …])</p></td>
<td><p>Compute the Matthews correlation coefficient (MCC)</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="generated/sklearn.metrics.roc_auc_score.html#sklearn.metrics.roc_auc_score" title="sklearn.metrics.roc_auc_score"><code class="xref py py-obj docutils literal notranslate"><span class="pre">roc_auc_score</span></code></a>(y_true, y_score[, average, …])</p></td>
<td><p>Compute Area Under the Receiver Operating Characteristic Curve (ROC AUC) from prediction scores.</p></td>
</tr>
</tbody>
</table>
<p>Some also work in the multilabel case:</p>
<table class="longtable docutils align-default">
<colgroup>
<col style="width: 10%" />
<col style="width: 90%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p><a class="reference internal" href="generated/sklearn.metrics.accuracy_score.html#sklearn.metrics.accuracy_score" title="sklearn.metrics.accuracy_score"><code class="xref py py-obj docutils literal notranslate"><span class="pre">accuracy_score</span></code></a>(y_true, y_pred[, normalize, …])</p></td>
<td><p>Accuracy classification score.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="generated/sklearn.metrics.classification_report.html#sklearn.metrics.classification_report" title="sklearn.metrics.classification_report"><code class="xref py py-obj docutils literal notranslate"><span class="pre">classification_report</span></code></a>(y_true, y_pred[, …])</p></td>
<td><p>Build a text report showing the main classification metrics</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="generated/sklearn.metrics.f1_score.html#sklearn.metrics.f1_score" title="sklearn.metrics.f1_score"><code class="xref py py-obj docutils literal notranslate"><span class="pre">f1_score</span></code></a>(y_true, y_pred[, labels, …])</p></td>
<td><p>Compute the F1 score, also known as balanced F-score or F-measure</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="generated/sklearn.metrics.fbeta_score.html#sklearn.metrics.fbeta_score" title="sklearn.metrics.fbeta_score"><code class="xref py py-obj docutils literal notranslate"><span class="pre">fbeta_score</span></code></a>(y_true, y_pred, beta[, labels, …])</p></td>
<td><p>Compute the F-beta score</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="generated/sklearn.metrics.hamming_loss.html#sklearn.metrics.hamming_loss" title="sklearn.metrics.hamming_loss"><code class="xref py py-obj docutils literal notranslate"><span class="pre">hamming_loss</span></code></a>(y_true, y_pred[, labels, …])</p></td>
<td><p>Compute the average Hamming loss.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="generated/sklearn.metrics.jaccard_score.html#sklearn.metrics.jaccard_score" title="sklearn.metrics.jaccard_score"><code class="xref py py-obj docutils literal notranslate"><span class="pre">jaccard_score</span></code></a>(y_true, y_pred[, labels, …])</p></td>
<td><p>Jaccard similarity coefficient score</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="generated/sklearn.metrics.log_loss.html#sklearn.metrics.log_loss" title="sklearn.metrics.log_loss"><code class="xref py py-obj docutils literal notranslate"><span class="pre">log_loss</span></code></a>(y_true, y_pred[, eps, normalize, …])</p></td>
<td><p>Log loss, aka logistic loss or cross-entropy loss.</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="generated/sklearn.metrics.multilabel_confusion_matrix.html#sklearn.metrics.multilabel_confusion_matrix" title="sklearn.metrics.multilabel_confusion_matrix"><code class="xref py py-obj docutils literal notranslate"><span class="pre">multilabel_confusion_matrix</span></code></a>(y_true, y_pred)</p></td>
<td><p>Compute a confusion matrix for each class or sample</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="generated/sklearn.metrics.precision_recall_fscore_support.html#sklearn.metrics.precision_recall_fscore_support" title="sklearn.metrics.precision_recall_fscore_support"><code class="xref py py-obj docutils literal notranslate"><span class="pre">precision_recall_fscore_support</span></code></a>(y_true, y_pred)</p></td>
<td><p>Compute precision, recall, F-measure and support for each class</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="generated/sklearn.metrics.precision_score.html#sklearn.metrics.precision_score" title="sklearn.metrics.precision_score"><code class="xref py py-obj docutils literal notranslate"><span class="pre">precision_score</span></code></a>(y_true, y_pred[, labels, …])</p></td>
<td><p>Compute the precision</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="generated/sklearn.metrics.recall_score.html#sklearn.metrics.recall_score" title="sklearn.metrics.recall_score"><code class="xref py py-obj docutils literal notranslate"><span class="pre">recall_score</span></code></a>(y_true, y_pred[, labels, …])</p></td>
<td><p>Compute the recall</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="generated/sklearn.metrics.roc_auc_score.html#sklearn.metrics.roc_auc_score" title="sklearn.metrics.roc_auc_score"><code class="xref py py-obj docutils literal notranslate"><span class="pre">roc_auc_score</span></code></a>(y_true, y_score[, average, …])</p></td>
<td><p>Compute Area Under the Receiver Operating Characteristic Curve (ROC AUC) from prediction scores.</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="generated/sklearn.metrics.zero_one_loss.html#sklearn.metrics.zero_one_loss" title="sklearn.metrics.zero_one_loss"><code class="xref py py-obj docutils literal notranslate"><span class="pre">zero_one_loss</span></code></a>(y_true, y_pred[, normalize, …])</p></td>
<td><p>Zero-one classification loss.</p></td>
</tr>
</tbody>
</table>
<p>And some work with binary and multilabel (but not multiclass) problems:</p>
<table class="longtable docutils align-default">
<colgroup>
<col style="width: 10%" />
<col style="width: 90%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p><a class="reference internal" href="generated/sklearn.metrics.average_precision_score.html#sklearn.metrics.average_precision_score" title="sklearn.metrics.average_precision_score"><code class="xref py py-obj docutils literal notranslate"><span class="pre">average_precision_score</span></code></a>(y_true, y_score[, …])</p></td>
<td><p>Compute average precision (AP) from prediction scores</p></td>
</tr>
</tbody>
</table>
<p>In the following sub-sections, we will describe each of those functions,
preceded by some notes on common API and metric definition.</p>
<div class="section" id="from-binary-to-multiclass-and-multilabel">
<span id="average"></span><h3>3.3.2.1. From binary to multiclass and multilabel<a class="headerlink" href="#from-binary-to-multiclass-and-multilabel" title="Permalink to this headline">¶</a></h3>
<p>Some metrics are essentially defined for binary classification tasks (e.g.
<a class="reference internal" href="generated/sklearn.metrics.f1_score.html#sklearn.metrics.f1_score" title="sklearn.metrics.f1_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">f1_score</span></code></a>, <a class="reference internal" href="generated/sklearn.metrics.roc_auc_score.html#sklearn.metrics.roc_auc_score" title="sklearn.metrics.roc_auc_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">roc_auc_score</span></code></a>). In these cases, by default
only the positive label is evaluated, assuming by default that the positive
class is labelled <code class="docutils literal notranslate"><span class="pre">1</span></code> (though this may be configurable through the
<code class="docutils literal notranslate"><span class="pre">pos_label</span></code> parameter).</p>
<p>In extending a binary metric to multiclass or multilabel problems, the data
is treated as a collection of binary problems, one for each class.
There are then a number of ways to average binary metric calculations across
the set of classes, each of which may be useful in some scenario.
Where available, you should select among these using the <code class="docutils literal notranslate"><span class="pre">average</span></code> parameter.</p>
<ul class="simple">
<li><p><code class="docutils literal notranslate"><span class="pre">&quot;macro&quot;</span></code> simply calculates the mean of the binary metrics,
giving equal weight to each class.  In problems where infrequent classes
are nonetheless important, macro-averaging may be a means of highlighting
their performance. On the other hand, the assumption that all classes are
equally important is often untrue, such that macro-averaging will
over-emphasize the typically low performance on an infrequent class.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">&quot;weighted&quot;</span></code> accounts for class imbalance by computing the average of
binary metrics in which each class’s score is weighted by its presence in the
true data sample.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">&quot;micro&quot;</span></code> gives each sample-class pair an equal contribution to the overall
metric (except as a result of sample-weight). Rather than summing the
metric per class, this sums the dividends and divisors that make up the
per-class metrics to calculate an overall quotient.
Micro-averaging may be preferred in multilabel settings, including
multiclass classification where a majority class is to be ignored.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">&quot;samples&quot;</span></code> applies only to multilabel problems. It does not calculate a
per-class measure, instead calculating the metric over the true and predicted
classes for each sample in the evaluation data, and returning their
(<code class="docutils literal notranslate"><span class="pre">sample_weight</span></code>-weighted) average.</p></li>
<li><p>Selecting <code class="docutils literal notranslate"><span class="pre">average=None</span></code> will return an array with the score for each
class.</p></li>
</ul>
<p>While multiclass data is provided to the metric, like binary targets, as an
array of class labels, multilabel data is specified as an indicator matrix,
in which cell <code class="docutils literal notranslate"><span class="pre">[i,</span> <span class="pre">j]</span></code> has value 1 if sample <code class="docutils literal notranslate"><span class="pre">i</span></code> has label <code class="docutils literal notranslate"><span class="pre">j</span></code> and value
0 otherwise.</p>
</div>
<div class="section" id="accuracy-score">
<span id="id2"></span><h3>3.3.2.2. Accuracy score<a class="headerlink" href="#accuracy-score" title="Permalink to this headline">¶</a></h3>
<p>The <a class="reference internal" href="generated/sklearn.metrics.accuracy_score.html#sklearn.metrics.accuracy_score" title="sklearn.metrics.accuracy_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">accuracy_score</span></code></a> function computes the
<a class="reference external" href="https://en.wikipedia.org/wiki/Accuracy_and_precision">accuracy</a>, either the fraction
(default) or the count (normalize=False) of correct predictions.</p>
<p>In multilabel classification, the function returns the subset accuracy. If
the entire set of predicted labels for a sample strictly match with the true
set of labels, then the subset accuracy is 1.0; otherwise it is 0.0.</p>
<p>If <span class="math notranslate nohighlight">\(\hat{y}_i\)</span> is the predicted value of
the <span class="math notranslate nohighlight">\(i\)</span>-th sample and <span class="math notranslate nohighlight">\(y_i\)</span> is the corresponding true value,
then the fraction of correct predictions over <span class="math notranslate nohighlight">\(n_\text{samples}\)</span> is
defined as</p>
<div class="math notranslate nohighlight">
\[\texttt{accuracy}(y, \hat{y}) = \frac{1}{n_\text{samples}} \sum_{i=0}^{n_\text{samples}-1} 1(\hat{y}_i = y_i)\]</div>
<p>where <span class="math notranslate nohighlight">\(1(x)\)</span> is the <a class="reference external" href="https://en.wikipedia.org/wiki/Indicator_function">indicator function</a>.</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="kn">import</span> <span class="n">accuracy_score</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_pred</span> <span class="o">=</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">3</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_true</span> <span class="o">=</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">accuracy_score</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">)</span>
<span class="go">0.5</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">accuracy_score</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">,</span> <span class="n">normalize</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
<span class="go">2</span>
</pre></div>
</div>
<p>In the multilabel case with binary label indicators:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">accuracy_score</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">]]),</span> <span class="n">np</span><span class="o">.</span><span class="n">ones</span><span class="p">((</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">)))</span>
<span class="go">0.5</span>
</pre></div>
</div>
<div class="topic">
<p class="topic-title">Example:</p>
<ul class="simple">
<li><p>See <a class="reference internal" href="../auto_examples/feature_selection/plot_permutation_test_for_classification.html#sphx-glr-auto-examples-feature-selection-plot-permutation-test-for-classification-py"><span class="std std-ref">Test with permutations the significance of a classification score</span></a>
for an example of accuracy score usage using permutations of
the dataset.</p></li>
</ul>
</div>
</div>
<div class="section" id="balanced-accuracy-score">
<span id="id3"></span><h3>3.3.2.3. Balanced accuracy score<a class="headerlink" href="#balanced-accuracy-score" title="Permalink to this headline">¶</a></h3>
<p>The <a class="reference internal" href="generated/sklearn.metrics.balanced_accuracy_score.html#sklearn.metrics.balanced_accuracy_score" title="sklearn.metrics.balanced_accuracy_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">balanced_accuracy_score</span></code></a> function computes the <a class="reference external" href="https://en.wikipedia.org/wiki/Accuracy_and_precision">balanced accuracy</a>, which avoids inflated
performance estimates on imbalanced datasets. It is the macro-average of recall
scores per class or, equivalently, raw accuracy where each sample is weighted
according to the inverse prevalence of its true class.
Thus for balanced datasets, the score is equal to accuracy.</p>
<p>In the binary case, balanced accuracy is equal to the arithmetic mean of
<a class="reference external" href="https://en.wikipedia.org/wiki/Sensitivity_and_specificity">sensitivity</a>
(true positive rate) and <a class="reference external" href="https://en.wikipedia.org/wiki/Sensitivity_and_specificity">specificity</a> (true negative
rate), or the area under the ROC curve with binary predictions rather than
scores.</p>
<p>If the classifier performs equally well on either class, this term reduces to
the conventional accuracy (i.e., the number of correct predictions divided by
the total number of predictions).</p>
<p>In contrast, if the conventional accuracy is above chance only because the
classifier takes advantage of an imbalanced test set, then the balanced
accuracy, as appropriate, will drop to <span class="math notranslate nohighlight">\(\frac{1}{n\_classes}\)</span>.</p>
<p>The score ranges from 0 to 1, or when <code class="docutils literal notranslate"><span class="pre">adjusted=True</span></code> is used, it rescaled to
the range <span class="math notranslate nohighlight">\(\frac{1}{1 - n\_classes}\)</span> to 1, inclusive, with
performance at random scoring 0.</p>
<p>If <span class="math notranslate nohighlight">\(y_i\)</span> is the true value of the <span class="math notranslate nohighlight">\(i\)</span>-th sample, and <span class="math notranslate nohighlight">\(w_i\)</span>
is the corresponding sample weight, then we adjust the sample weight to:</p>
<div class="math notranslate nohighlight">
\[\hat{w}_i = \frac{w_i}{\sum_j{1(y_j = y_i) w_j}}\]</div>
<p>where <span class="math notranslate nohighlight">\(1(x)\)</span> is the <a class="reference external" href="https://en.wikipedia.org/wiki/Indicator_function">indicator function</a>.
Given predicted <span class="math notranslate nohighlight">\(\hat{y}_i\)</span> for sample <span class="math notranslate nohighlight">\(i\)</span>, balanced accuracy is
defined as:</p>
<div class="math notranslate nohighlight">
\[\texttt{balanced-accuracy}(y, \hat{y}, w) = \frac{1}{\sum{\hat{w}_i}} \sum_i 1(\hat{y}_i = y_i) \hat{w}_i\]</div>
<p>With <code class="docutils literal notranslate"><span class="pre">adjusted=True</span></code>, balanced accuracy reports the relative increase from
<span class="math notranslate nohighlight">\(\texttt{balanced-accuracy}(y, \mathbf{0}, w) =
\frac{1}{n\_classes}\)</span>.  In the binary case, this is also known as
<a class="reference external" href="https://en.wikipedia.org/wiki/Youden%27s_J_statistic">*Youden’s J statistic*</a>,
or <em>informedness</em>.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>The multiclass definition here seems the most reasonable extension of the
metric used in binary classification, though there is no certain consensus
in the literature:</p>
<ul class="simple">
<li><p>Our definition: <a class="reference internal" href="#mosley2013" id="id5"><span>[Mosley2013]</span></a>, <a class="reference internal" href="#kelleher2015" id="id6"><span>[Kelleher2015]</span></a> and <a class="reference internal" href="#guyon2015" id="id7"><span>[Guyon2015]</span></a>, where
<a class="reference internal" href="#guyon2015" id="id8"><span>[Guyon2015]</span></a> adopt the adjusted version to ensure that random predictions
have a score of <span class="math notranslate nohighlight">\(0\)</span> and perfect predictions have a score of <span class="math notranslate nohighlight">\(1\)</span>..</p></li>
<li><p>Class balanced accuracy as described in <a class="reference internal" href="#mosley2013" id="id9"><span>[Mosley2013]</span></a>: the minimum between the precision
and the recall for each class is computed. Those values are then averaged over the total
number of classes to get the balanced accuracy.</p></li>
<li><p>Balanced Accuracy as described in <a class="reference internal" href="#urbanowicz2015" id="id10"><span>[Urbanowicz2015]</span></a>: the average of sensitivity and specificity
is computed for each class and then averaged over total number of classes.</p></li>
</ul>
</div>
<div class="topic">
<p class="topic-title">References:</p>
<dl class="citation">
<dt class="label" id="guyon2015"><span class="brackets">Guyon2015</span><span class="fn-backref">(<a href="#id7">1</a>,<a href="#id8">2</a>)</span></dt>
<dd><p>I. Guyon, K. Bennett, G. Cawley, H.J. Escalante, S. Escalera, T.K. Ho, N. Macià,
B. Ray, M. Saeed, A.R. Statnikov, E. Viegas, <a class="reference external" href="https://ieeexplore.ieee.org/document/7280767">Design of the 2015 ChaLearn AutoML Challenge</a>,
IJCNN 2015.</p>
</dd>
<dt class="label" id="mosley2013"><span class="brackets">Mosley2013</span><span class="fn-backref">(<a href="#id5">1</a>,<a href="#id9">2</a>)</span></dt>
<dd><p>L. Mosley, <a class="reference external" href="https://lib.dr.iastate.edu/etd/13537/">A balanced approach to the multi-class imbalance problem</a>,
IJCV 2010.</p>
</dd>
<dt class="label" id="kelleher2015"><span class="brackets"><a class="fn-backref" href="#id6">Kelleher2015</a></span></dt>
<dd><p>John. D. Kelleher, Brian Mac Namee, Aoife D’Arcy, <a class="reference external" href="https://mitpress.mit.edu/books/fundamentals-machine-learning-predictive-data-analytics">Fundamentals of
Machine Learning for Predictive Data Analytics: Algorithms, Worked Examples,
and Case Studies</a>,
2015.</p>
</dd>
<dt class="label" id="urbanowicz2015"><span class="brackets"><a class="fn-backref" href="#id10">Urbanowicz2015</a></span></dt>
<dd><p>Urbanowicz R.J.,  Moore, J.H. <a class="reference external" href="https://doi.org/10.1007/s12065-015-0128-8">ExSTraCS 2.0: description and evaluation of a scalable learning
classifier system</a>, Evol. Intel. (2015) 8: 89.</p>
</dd>
</dl>
</div>
</div>
<div class="section" id="cohen-s-kappa">
<span id="cohen-kappa"></span><h3>3.3.2.4. Cohen’s kappa<a class="headerlink" href="#cohen-s-kappa" title="Permalink to this headline">¶</a></h3>
<p>The function <a class="reference internal" href="generated/sklearn.metrics.cohen_kappa_score.html#sklearn.metrics.cohen_kappa_score" title="sklearn.metrics.cohen_kappa_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">cohen_kappa_score</span></code></a> computes <a class="reference external" href="https://en.wikipedia.org/wiki/Cohen%27s_kappa">Cohen’s kappa</a> statistic.
This measure is intended to compare labelings by different human annotators,
not a classifier versus a ground truth.</p>
<p>The kappa score (see docstring) is a number between -1 and 1.
Scores above .8 are generally considered good agreement;
zero or lower means no agreement (practically random labels).</p>
<p>Kappa scores can be computed for binary or multiclass problems,
but not for multilabel problems (except by manually computing a per-label score)
and not for more than two annotators.</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="kn">import</span> <span class="n">cohen_kappa_score</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_true</span> <span class="o">=</span> <span class="p">[</span><span class="mi">2</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_pred</span> <span class="o">=</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">2</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">cohen_kappa_score</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">)</span>
<span class="go">0.4285714285714286</span>
</pre></div>
</div>
</div>
<div class="section" id="confusion-matrix">
<span id="id12"></span><h3>3.3.2.5. Confusion matrix<a class="headerlink" href="#confusion-matrix" title="Permalink to this headline">¶</a></h3>
<p>The <a class="reference internal" href="generated/sklearn.metrics.confusion_matrix.html#sklearn.metrics.confusion_matrix" title="sklearn.metrics.confusion_matrix"><code class="xref py py-func docutils literal notranslate"><span class="pre">confusion_matrix</span></code></a> function evaluates
classification accuracy by computing the confusion matrix
with each row corresponding to the true class
&lt;<a class="reference external" href="https://en.wikipedia.org/wiki/Confusion_matrix">https://en.wikipedia.org/wiki/Confusion_matrix</a>&gt;`_.
(Wikipedia and other references may use different convention for axes.)</p>
<p>By definition, entry <span class="math notranslate nohighlight">\(i, j\)</span> in a confusion matrix is
the number of observations actually in group <span class="math notranslate nohighlight">\(i\)</span>, but
predicted to be in group <span class="math notranslate nohighlight">\(j\)</span>. Here is an example:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="kn">import</span> <span class="n">confusion_matrix</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_true</span> <span class="o">=</span> <span class="p">[</span><span class="mi">2</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_pred</span> <span class="o">=</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">2</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">confusion_matrix</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">)</span>
<span class="go">array([[2, 0, 0],</span>
<span class="go">       [0, 0, 1],</span>
<span class="go">       [1, 0, 2]])</span>
</pre></div>
</div>
<p><a class="reference internal" href="generated/sklearn.metrics.plot_confusion_matrix.html#sklearn.metrics.plot_confusion_matrix" title="sklearn.metrics.plot_confusion_matrix"><code class="xref py py-func docutils literal notranslate"><span class="pre">plot_confusion_matrix</span></code></a> can be used to visually represent a confusion
matrix as shown in the
<a class="reference internal" href="../auto_examples/model_selection/plot_confusion_matrix.html#sphx-glr-auto-examples-model-selection-plot-confusion-matrix-py"><span class="std std-ref">Confusion matrix</span></a>
example, which creates the following figure:</p>
<a class="reference external image-reference" href="../auto_examples/model_selection/plot_confusion_matrix.html"><img alt="modules/../auto_examples/model_selection/images/sphx_glr_plot_confusion_matrix_001.png" class="align-center" src="modules/../auto_examples/model_selection/images/sphx_glr_plot_confusion_matrix_001.png" /></a>
<p>The parameter <code class="docutils literal notranslate"><span class="pre">normalize</span></code> allows to report ratios instead of counts. The
confusion matrix can be normalized in 3 different ways: <code class="docutils literal notranslate"><span class="pre">'pred'</span></code>, <code class="docutils literal notranslate"><span class="pre">'true'</span></code>,
and <code class="docutils literal notranslate"><span class="pre">'all'</span></code> which will divide the counts by the sum of each columns, rows, or
the entire matrix, respectively.</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">y_true</span> <span class="o">=</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_pred</span> <span class="o">=</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">confusion_matrix</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">,</span> <span class="n">normalize</span><span class="o">=</span><span class="s1">&#39;all&#39;</span><span class="p">)</span>
<span class="go">array([[0.25 , 0.125],</span>
<span class="go">       [0.25 , 0.375]])</span>
</pre></div>
</div>
<p>For binary problems, we can get counts of true negatives, false positives,
false negatives and true positives as follows:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">y_true</span> <span class="o">=</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_pred</span> <span class="o">=</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">tn</span><span class="p">,</span> <span class="n">fp</span><span class="p">,</span> <span class="n">fn</span><span class="p">,</span> <span class="n">tp</span> <span class="o">=</span> <span class="n">confusion_matrix</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">)</span><span class="o">.</span><span class="n">ravel</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">tn</span><span class="p">,</span> <span class="n">fp</span><span class="p">,</span> <span class="n">fn</span><span class="p">,</span> <span class="n">tp</span>
<span class="go">(2, 1, 2, 3)</span>
</pre></div>
</div>
<div class="topic">
<p class="topic-title">Example:</p>
<ul class="simple">
<li><p>See <a class="reference internal" href="../auto_examples/model_selection/plot_confusion_matrix.html#sphx-glr-auto-examples-model-selection-plot-confusion-matrix-py"><span class="std std-ref">Confusion matrix</span></a>
for an example of using a confusion matrix to evaluate classifier output
quality.</p></li>
<li><p>See <a class="reference internal" href="../auto_examples/classification/plot_digits_classification.html#sphx-glr-auto-examples-classification-plot-digits-classification-py"><span class="std std-ref">Recognizing hand-written digits</span></a>
for an example of using a confusion matrix to classify
hand-written digits.</p></li>
<li><p>See <a class="reference internal" href="../auto_examples/text/plot_document_classification_20newsgroups.html#sphx-glr-auto-examples-text-plot-document-classification-20newsgroups-py"><span class="std std-ref">Classification of text documents using sparse features</span></a>
for an example of using a confusion matrix to classify text
documents.</p></li>
</ul>
</div>
</div>
<div class="section" id="classification-report">
<span id="id13"></span><h3>3.3.2.6. Classification report<a class="headerlink" href="#classification-report" title="Permalink to this headline">¶</a></h3>
<p>The <a class="reference internal" href="generated/sklearn.metrics.classification_report.html#sklearn.metrics.classification_report" title="sklearn.metrics.classification_report"><code class="xref py py-func docutils literal notranslate"><span class="pre">classification_report</span></code></a> function builds a text report showing the
main classification metrics. Here is a small example with custom <code class="docutils literal notranslate"><span class="pre">target_names</span></code>
and inferred labels:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="kn">import</span> <span class="n">classification_report</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_true</span> <span class="o">=</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">0</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_pred</span> <span class="o">=</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">target_names</span> <span class="o">=</span> <span class="p">[</span><span class="s1">&#39;class 0&#39;</span><span class="p">,</span> <span class="s1">&#39;class 1&#39;</span><span class="p">,</span> <span class="s1">&#39;class 2&#39;</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="nb">print</span><span class="p">(</span><span class="n">classification_report</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">,</span> <span class="n">target_names</span><span class="o">=</span><span class="n">target_names</span><span class="p">))</span>
<span class="go">              precision    recall  f1-score   support</span>

<span class="go">     class 0       0.67      1.00      0.80         2</span>
<span class="go">     class 1       0.00      0.00      0.00         1</span>
<span class="go">     class 2       1.00      0.50      0.67         2</span>

<span class="go">    accuracy                           0.60         5</span>
<span class="go">   macro avg       0.56      0.50      0.49         5</span>
<span class="go">weighted avg       0.67      0.60      0.59         5</span>
</pre></div>
</div>
<div class="topic">
<p class="topic-title">Example:</p>
<ul class="simple">
<li><p>See <a class="reference internal" href="../auto_examples/classification/plot_digits_classification.html#sphx-glr-auto-examples-classification-plot-digits-classification-py"><span class="std std-ref">Recognizing hand-written digits</span></a>
for an example of classification report usage for
hand-written digits.</p></li>
<li><p>See <a class="reference internal" href="../auto_examples/text/plot_document_classification_20newsgroups.html#sphx-glr-auto-examples-text-plot-document-classification-20newsgroups-py"><span class="std std-ref">Classification of text documents using sparse features</span></a>
for an example of classification report usage for text
documents.</p></li>
<li><p>See <a class="reference internal" href="../auto_examples/model_selection/plot_grid_search_digits.html#sphx-glr-auto-examples-model-selection-plot-grid-search-digits-py"><span class="std std-ref">Parameter estimation using grid search with cross-validation</span></a>
for an example of classification report usage for
grid search with nested cross-validation.</p></li>
</ul>
</div>
</div>
<div class="section" id="hamming-loss">
<span id="id14"></span><h3>3.3.2.7. Hamming loss<a class="headerlink" href="#hamming-loss" title="Permalink to this headline">¶</a></h3>
<p>The <a class="reference internal" href="generated/sklearn.metrics.hamming_loss.html#sklearn.metrics.hamming_loss" title="sklearn.metrics.hamming_loss"><code class="xref py py-func docutils literal notranslate"><span class="pre">hamming_loss</span></code></a> computes the average Hamming loss or <a class="reference external" href="https://en.wikipedia.org/wiki/Hamming_distance">Hamming
distance</a> between two sets
of samples.</p>
<p>If <span class="math notranslate nohighlight">\(\hat{y}_j\)</span> is the predicted value for the <span class="math notranslate nohighlight">\(j\)</span>-th label of
a given sample, <span class="math notranslate nohighlight">\(y_j\)</span> is the corresponding true value, and
<span class="math notranslate nohighlight">\(n_\text{labels}\)</span> is the number of classes or labels, then the
Hamming loss <span class="math notranslate nohighlight">\(L_{Hamming}\)</span> between two samples is defined as:</p>
<div class="math notranslate nohighlight">
\[L_{Hamming}(y, \hat{y}) = \frac{1}{n_\text{labels}} \sum_{j=0}^{n_\text{labels} - 1} 1(\hat{y}_j \not= y_j)\]</div>
<p>where <span class="math notranslate nohighlight">\(1(x)\)</span> is the <a class="reference external" href="https://en.wikipedia.org/wiki/Indicator_function">indicator function</a>.</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="kn">import</span> <span class="n">hamming_loss</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_pred</span> <span class="o">=</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_true</span> <span class="o">=</span> <span class="p">[</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">hamming_loss</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">)</span>
<span class="go">0.25</span>
</pre></div>
</div>
<p>In the multilabel case with binary label indicators:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">hamming_loss</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">]]),</span> <span class="n">np</span><span class="o">.</span><span class="n">zeros</span><span class="p">((</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">)))</span>
<span class="go">0.75</span>
</pre></div>
</div>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>In multiclass classification, the Hamming loss corresponds to the Hamming
distance between <code class="docutils literal notranslate"><span class="pre">y_true</span></code> and <code class="docutils literal notranslate"><span class="pre">y_pred</span></code> which is similar to the
<a class="reference internal" href="#zero-one-loss"><span class="std std-ref">Zero one loss</span></a> function.  However, while zero-one loss penalizes
prediction sets that do not strictly match true sets, the Hamming loss
penalizes individual labels.  Thus the Hamming loss, upper bounded by the zero-one
loss, is always between zero and one, inclusive; and predicting a proper subset
or superset of the true labels will give a Hamming loss between
zero and one, exclusive.</p>
</div>
</div>
<div class="section" id="precision-recall-and-f-measures">
<span id="precision-recall-f-measure-metrics"></span><h3>3.3.2.8. Precision, recall and F-measures<a class="headerlink" href="#precision-recall-and-f-measures" title="Permalink to this headline">¶</a></h3>
<p>Intuitively, <a class="reference external" href="https://en.wikipedia.org/wiki/Precision_and_recall#Precision">precision</a> is the ability
of the classifier not to label as positive a sample that is negative, and
<a class="reference external" href="https://en.wikipedia.org/wiki/Precision_and_recall#Recall">recall</a> is the
ability of the classifier to find all the positive samples.</p>
<p>The  <a class="reference external" href="https://en.wikipedia.org/wiki/F1_score">F-measure</a>
(<span class="math notranslate nohighlight">\(F_\beta\)</span> and <span class="math notranslate nohighlight">\(F_1\)</span> measures) can be interpreted as a weighted
harmonic mean of the precision and recall. A
<span class="math notranslate nohighlight">\(F_\beta\)</span> measure reaches its best value at 1 and its worst score at 0.
With <span class="math notranslate nohighlight">\(\beta = 1\)</span>,  <span class="math notranslate nohighlight">\(F_\beta\)</span> and
<span class="math notranslate nohighlight">\(F_1\)</span>  are equivalent, and the recall and the precision are equally important.</p>
<p>The <a class="reference internal" href="generated/sklearn.metrics.precision_recall_curve.html#sklearn.metrics.precision_recall_curve" title="sklearn.metrics.precision_recall_curve"><code class="xref py py-func docutils literal notranslate"><span class="pre">precision_recall_curve</span></code></a> computes a precision-recall curve
from the ground truth label and a score given by the classifier
by varying a decision threshold.</p>
<p>The <a class="reference internal" href="generated/sklearn.metrics.average_precision_score.html#sklearn.metrics.average_precision_score" title="sklearn.metrics.average_precision_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">average_precision_score</span></code></a> function computes the
<a class="reference external" href="https://en.wikipedia.org/w/index.php?title=Information_retrieval&amp;oldid=793358396#Average_precision">average precision</a>
(AP) from prediction scores. The value is between 0 and 1 and higher is better.
AP is defined as</p>
<div class="math notranslate nohighlight">
\[\text{AP} = \sum_n (R_n - R_{n-1}) P_n\]</div>
<p>where <span class="math notranslate nohighlight">\(P_n\)</span> and <span class="math notranslate nohighlight">\(R_n\)</span> are the precision and recall at the
nth threshold. With random predictions, the AP is the fraction of positive
samples.</p>
<p>References <a class="reference internal" href="#manning2008" id="id16"><span>[Manning2008]</span></a> and <a class="reference internal" href="#everingham2010" id="id17"><span>[Everingham2010]</span></a> present alternative variants of
AP that interpolate the precision-recall curve. Currently,
<a class="reference internal" href="generated/sklearn.metrics.average_precision_score.html#sklearn.metrics.average_precision_score" title="sklearn.metrics.average_precision_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">average_precision_score</span></code></a> does not implement any interpolated variant.
References <a class="reference internal" href="#davis2006" id="id18"><span>[Davis2006]</span></a> and <a class="reference internal" href="#flach2015" id="id19"><span>[Flach2015]</span></a> describe why a linear interpolation of
points on the precision-recall curve provides an overly-optimistic measure of
classifier performance. This linear interpolation is used when computing area
under the curve with the trapezoidal rule in <a class="reference internal" href="generated/sklearn.metrics.auc.html#sklearn.metrics.auc" title="sklearn.metrics.auc"><code class="xref py py-func docutils literal notranslate"><span class="pre">auc</span></code></a>.</p>
<p>Several functions allow you to analyze the precision, recall and F-measures
score:</p>
<table class="longtable docutils align-default">
<colgroup>
<col style="width: 10%" />
<col style="width: 90%" />
</colgroup>
<tbody>
<tr class="row-odd"><td><p><a class="reference internal" href="generated/sklearn.metrics.average_precision_score.html#sklearn.metrics.average_precision_score" title="sklearn.metrics.average_precision_score"><code class="xref py py-obj docutils literal notranslate"><span class="pre">average_precision_score</span></code></a>(y_true, y_score[, …])</p></td>
<td><p>Compute average precision (AP) from prediction scores</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="generated/sklearn.metrics.f1_score.html#sklearn.metrics.f1_score" title="sklearn.metrics.f1_score"><code class="xref py py-obj docutils literal notranslate"><span class="pre">f1_score</span></code></a>(y_true, y_pred[, labels, …])</p></td>
<td><p>Compute the F1 score, also known as balanced F-score or F-measure</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="generated/sklearn.metrics.fbeta_score.html#sklearn.metrics.fbeta_score" title="sklearn.metrics.fbeta_score"><code class="xref py py-obj docutils literal notranslate"><span class="pre">fbeta_score</span></code></a>(y_true, y_pred, beta[, labels, …])</p></td>
<td><p>Compute the F-beta score</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="generated/sklearn.metrics.precision_recall_curve.html#sklearn.metrics.precision_recall_curve" title="sklearn.metrics.precision_recall_curve"><code class="xref py py-obj docutils literal notranslate"><span class="pre">precision_recall_curve</span></code></a>(y_true, probas_pred)</p></td>
<td><p>Compute precision-recall pairs for different probability thresholds</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="generated/sklearn.metrics.precision_recall_fscore_support.html#sklearn.metrics.precision_recall_fscore_support" title="sklearn.metrics.precision_recall_fscore_support"><code class="xref py py-obj docutils literal notranslate"><span class="pre">precision_recall_fscore_support</span></code></a>(y_true, y_pred)</p></td>
<td><p>Compute precision, recall, F-measure and support for each class</p></td>
</tr>
<tr class="row-even"><td><p><a class="reference internal" href="generated/sklearn.metrics.precision_score.html#sklearn.metrics.precision_score" title="sklearn.metrics.precision_score"><code class="xref py py-obj docutils literal notranslate"><span class="pre">precision_score</span></code></a>(y_true, y_pred[, labels, …])</p></td>
<td><p>Compute the precision</p></td>
</tr>
<tr class="row-odd"><td><p><a class="reference internal" href="generated/sklearn.metrics.recall_score.html#sklearn.metrics.recall_score" title="sklearn.metrics.recall_score"><code class="xref py py-obj docutils literal notranslate"><span class="pre">recall_score</span></code></a>(y_true, y_pred[, labels, …])</p></td>
<td><p>Compute the recall</p></td>
</tr>
</tbody>
</table>
<p>Note that the <a class="reference internal" href="generated/sklearn.metrics.precision_recall_curve.html#sklearn.metrics.precision_recall_curve" title="sklearn.metrics.precision_recall_curve"><code class="xref py py-func docutils literal notranslate"><span class="pre">precision_recall_curve</span></code></a> function is restricted to the
binary case. The <a class="reference internal" href="generated/sklearn.metrics.average_precision_score.html#sklearn.metrics.average_precision_score" title="sklearn.metrics.average_precision_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">average_precision_score</span></code></a> function works only in
binary classification and multilabel indicator format. The
<a class="reference internal" href="generated/sklearn.metrics.plot_precision_recall_curve.html#sklearn.metrics.plot_precision_recall_curve" title="sklearn.metrics.plot_precision_recall_curve"><code class="xref py py-func docutils literal notranslate"><span class="pre">plot_precision_recall_curve</span></code></a> function plots the precision recall as
follows.</p>
<a class="reference external image-reference" href="../auto_examples/model_selection/plot_precision_recall.html#plot-the-precision-recall-curve"><img alt="modules/../auto_examples/model_selection/images/sphx_glr_plot_precision_recall_001.png" class="align-center" src="modules/../auto_examples/model_selection/images/sphx_glr_plot_precision_recall_001.png" /></a>
<div class="topic">
<p class="topic-title">Examples:</p>
<ul class="simple">
<li><p>See <a class="reference internal" href="../auto_examples/text/plot_document_classification_20newsgroups.html#sphx-glr-auto-examples-text-plot-document-classification-20newsgroups-py"><span class="std std-ref">Classification of text documents using sparse features</span></a>
for an example of <a class="reference internal" href="generated/sklearn.metrics.f1_score.html#sklearn.metrics.f1_score" title="sklearn.metrics.f1_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">f1_score</span></code></a> usage to classify  text
documents.</p></li>
<li><p>See <a class="reference internal" href="../auto_examples/model_selection/plot_grid_search_digits.html#sphx-glr-auto-examples-model-selection-plot-grid-search-digits-py"><span class="std std-ref">Parameter estimation using grid search with cross-validation</span></a>
for an example of <a class="reference internal" href="generated/sklearn.metrics.precision_score.html#sklearn.metrics.precision_score" title="sklearn.metrics.precision_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">precision_score</span></code></a> and <a class="reference internal" href="generated/sklearn.metrics.recall_score.html#sklearn.metrics.recall_score" title="sklearn.metrics.recall_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">recall_score</span></code></a> usage
to estimate parameters using grid search with nested cross-validation.</p></li>
<li><p>See <a class="reference internal" href="../auto_examples/model_selection/plot_precision_recall.html#sphx-glr-auto-examples-model-selection-plot-precision-recall-py"><span class="std std-ref">Precision-Recall</span></a>
for an example of <a class="reference internal" href="generated/sklearn.metrics.precision_recall_curve.html#sklearn.metrics.precision_recall_curve" title="sklearn.metrics.precision_recall_curve"><code class="xref py py-func docutils literal notranslate"><span class="pre">precision_recall_curve</span></code></a> usage to evaluate
classifier output quality.</p></li>
</ul>
</div>
<div class="topic">
<p class="topic-title">References:</p>
<dl class="citation">
<dt class="label" id="manning2008"><span class="brackets"><a class="fn-backref" href="#id16">Manning2008</a></span></dt>
<dd><p>C.D. Manning, P. Raghavan, H. Schütze, <a class="reference external" href="https://nlp.stanford.edu/IR-book/html/htmledition/evaluation-of-ranked-retrieval-results-1.html">Introduction to Information Retrieval</a>,
2008.</p>
</dd>
<dt class="label" id="everingham2010"><span class="brackets"><a class="fn-backref" href="#id17">Everingham2010</a></span></dt>
<dd><p>M. Everingham, L. Van Gool, C.K.I. Williams, J. Winn, A. Zisserman,
<a class="reference external" href="http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.157.5766&amp;rep=rep1&amp;type=pdf">The Pascal Visual Object Classes (VOC) Challenge</a>,
IJCV 2010.</p>
</dd>
<dt class="label" id="davis2006"><span class="brackets"><a class="fn-backref" href="#id18">Davis2006</a></span></dt>
<dd><p>J. Davis, M. Goadrich, <a class="reference external" href="http://www.machinelearning.org/proceedings/icml2006/030_The_Relationship_Bet.pdf">The Relationship Between Precision-Recall and ROC Curves</a>,
ICML 2006.</p>
</dd>
<dt class="label" id="flach2015"><span class="brackets"><a class="fn-backref" href="#id19">Flach2015</a></span></dt>
<dd><p>P.A. Flach, M. Kull, <a class="reference external" href="https://papers.nips.cc/paper/5867-precision-recall-gain-curves-pr-analysis-done-right.pdf">Precision-Recall-Gain Curves: PR Analysis Done Right</a>,
NIPS 2015.</p>
</dd>
</dl>
</div>
<div class="section" id="binary-classification">
<h4>3.3.2.8.1. Binary classification<a class="headerlink" href="#binary-classification" title="Permalink to this headline">¶</a></h4>
<p>In a binary classification task, the terms ‘’positive’’ and ‘’negative’’ refer
to the classifier’s prediction, and the terms ‘’true’’ and ‘’false’’ refer to
whether that prediction corresponds to the external judgment (sometimes known
as the ‘’observation’’). Given these definitions, we can formulate the
following table:</p>
<table class="docutils align-default">
<colgroup>
<col style="width: 29%" />
<col style="width: 32%" />
<col style="width: 39%" />
</colgroup>
<tbody>
<tr class="row-odd"><td></td>
<td colspan="2"><p>Actual class (observation)</p></td>
</tr>
<tr class="row-even"><td rowspan="2"><p>Predicted class
(expectation)</p></td>
<td><p>tp (true positive)
Correct result</p></td>
<td><p>fp (false positive)
Unexpected result</p></td>
</tr>
<tr class="row-odd"><td><p>fn (false negative)
Missing result</p></td>
<td><p>tn (true negative)
Correct absence of result</p></td>
</tr>
</tbody>
</table>
<p>In this context, we can define the notions of precision, recall and F-measure:</p>
<div class="math notranslate nohighlight">
\[\text{precision} = \frac{tp}{tp + fp},\]</div>
<div class="math notranslate nohighlight">
\[\text{recall} = \frac{tp}{tp + fn},\]</div>
<div class="math notranslate nohighlight">
\[F_\beta = (1 + \beta^2) \frac{\text{precision} \times \text{recall}}{\beta^2 \text{precision} + \text{recall}}.\]</div>
<p>Here are some small examples in binary classification:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn</span> <span class="kn">import</span> <span class="n">metrics</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_pred</span> <span class="o">=</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_true</span> <span class="o">=</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">metrics</span><span class="o">.</span><span class="n">precision_score</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">)</span>
<span class="go">1.0</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">metrics</span><span class="o">.</span><span class="n">recall_score</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">)</span>
<span class="go">0.5</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">metrics</span><span class="o">.</span><span class="n">f1_score</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">)</span>
<span class="go">0.66...</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">metrics</span><span class="o">.</span><span class="n">fbeta_score</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">,</span> <span class="n">beta</span><span class="o">=</span><span class="mf">0.5</span><span class="p">)</span>
<span class="go">0.83...</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">metrics</span><span class="o">.</span><span class="n">fbeta_score</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">,</span> <span class="n">beta</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
<span class="go">0.66...</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">metrics</span><span class="o">.</span><span class="n">fbeta_score</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">,</span> <span class="n">beta</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
<span class="go">0.55...</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">metrics</span><span class="o">.</span><span class="n">precision_recall_fscore_support</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">,</span> <span class="n">beta</span><span class="o">=</span><span class="mf">0.5</span><span class="p">)</span>
<span class="go">(array([0.66..., 1.        ]), array([1. , 0.5]), array([0.71..., 0.83...]), array([2, 2]))</span>


<span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="kn">import</span> <span class="n">precision_recall_curve</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="kn">import</span> <span class="n">average_precision_score</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_true</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_scores</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mf">0.1</span><span class="p">,</span> <span class="mf">0.4</span><span class="p">,</span> <span class="mf">0.35</span><span class="p">,</span> <span class="mf">0.8</span><span class="p">])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">precision</span><span class="p">,</span> <span class="n">recall</span><span class="p">,</span> <span class="n">threshold</span> <span class="o">=</span> <span class="n">precision_recall_curve</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_scores</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">precision</span>
<span class="go">array([0.66..., 0.5       , 1.        , 1.        ])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">recall</span>
<span class="go">array([1. , 0.5, 0.5, 0. ])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">threshold</span>
<span class="go">array([0.35, 0.4 , 0.8 ])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">average_precision_score</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_scores</span><span class="p">)</span>
<span class="go">0.83...</span>
</pre></div>
</div>
</div>
<div class="section" id="multiclass-and-multilabel-classification">
<h4>3.3.2.8.2. Multiclass and multilabel classification<a class="headerlink" href="#multiclass-and-multilabel-classification" title="Permalink to this headline">¶</a></h4>
<p>In multiclass and multilabel classification task, the notions of precision,
recall, and F-measures can be applied to each label independently.
There are a few ways to combine results across labels,
specified by the <code class="docutils literal notranslate"><span class="pre">average</span></code> argument to the
<a class="reference internal" href="generated/sklearn.metrics.average_precision_score.html#sklearn.metrics.average_precision_score" title="sklearn.metrics.average_precision_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">average_precision_score</span></code></a> (multilabel only), <a class="reference internal" href="generated/sklearn.metrics.f1_score.html#sklearn.metrics.f1_score" title="sklearn.metrics.f1_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">f1_score</span></code></a>,
<a class="reference internal" href="generated/sklearn.metrics.fbeta_score.html#sklearn.metrics.fbeta_score" title="sklearn.metrics.fbeta_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">fbeta_score</span></code></a>, <a class="reference internal" href="generated/sklearn.metrics.precision_recall_fscore_support.html#sklearn.metrics.precision_recall_fscore_support" title="sklearn.metrics.precision_recall_fscore_support"><code class="xref py py-func docutils literal notranslate"><span class="pre">precision_recall_fscore_support</span></code></a>,
<a class="reference internal" href="generated/sklearn.metrics.precision_score.html#sklearn.metrics.precision_score" title="sklearn.metrics.precision_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">precision_score</span></code></a> and <a class="reference internal" href="generated/sklearn.metrics.recall_score.html#sklearn.metrics.recall_score" title="sklearn.metrics.recall_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">recall_score</span></code></a> functions, as described
<a class="reference internal" href="#average"><span class="std std-ref">above</span></a>. Note that if all labels are included, “micro”-averaging
in a multiclass setting will produce precision, recall and <span class="math notranslate nohighlight">\(F\)</span>
that are all identical to accuracy. Also note that “weighted” averaging may
produce an F-score that is not between precision and recall.</p>
<p>To make this more explicit, consider the following notation:</p>
<ul class="simple">
<li><p><span class="math notranslate nohighlight">\(y\)</span> the set of <em>predicted</em> <span class="math notranslate nohighlight">\((sample, label)\)</span> pairs</p></li>
<li><p><span class="math notranslate nohighlight">\(\hat{y}\)</span> the set of <em>true</em> <span class="math notranslate nohighlight">\((sample, label)\)</span> pairs</p></li>
<li><p><span class="math notranslate nohighlight">\(L\)</span> the set of labels</p></li>
<li><p><span class="math notranslate nohighlight">\(S\)</span> the set of samples</p></li>
<li><p><span class="math notranslate nohighlight">\(y_s\)</span> the subset of <span class="math notranslate nohighlight">\(y\)</span> with sample <span class="math notranslate nohighlight">\(s\)</span>,
i.e. <span class="math notranslate nohighlight">\(y_s := \left\{(s', l) \in y | s' = s\right\}\)</span></p></li>
<li><p><span class="math notranslate nohighlight">\(y_l\)</span> the subset of <span class="math notranslate nohighlight">\(y\)</span> with label <span class="math notranslate nohighlight">\(l\)</span></p></li>
<li><p>similarly, <span class="math notranslate nohighlight">\(\hat{y}_s\)</span> and <span class="math notranslate nohighlight">\(\hat{y}_l\)</span> are subsets of
<span class="math notranslate nohighlight">\(\hat{y}\)</span></p></li>
<li><p><span class="math notranslate nohighlight">\(P(A, B) := \frac{\left| A \cap B \right|}{\left|A\right|}\)</span> for some
sets <span class="math notranslate nohighlight">\(A\)</span> and <span class="math notranslate nohighlight">\(B\)</span></p></li>
<li><p><span class="math notranslate nohighlight">\(R(A, B) := \frac{\left| A \cap B \right|}{\left|B\right|}\)</span>
(Conventions vary on handling <span class="math notranslate nohighlight">\(B = \emptyset\)</span>; this implementation uses
<span class="math notranslate nohighlight">\(R(A, B):=0\)</span>, and similar for <span class="math notranslate nohighlight">\(P\)</span>.)</p></li>
<li><p><span class="math notranslate nohighlight">\(F_\beta(A, B) := \left(1 + \beta^2\right) \frac{P(A, B) \times R(A, B)}{\beta^2 P(A, B) + R(A, B)}\)</span></p></li>
</ul>
<p>Then the metrics are defined as:</p>
<table class="docutils align-default">
<colgroup>
<col style="width: 4%" />
<col style="width: 32%" />
<col style="width: 32%" />
<col style="width: 33%" />
</colgroup>
<thead>
<tr class="row-odd"><th class="head"><p><code class="docutils literal notranslate"><span class="pre">average</span></code></p></th>
<th class="head"><p>Precision</p></th>
<th class="head"><p>Recall</p></th>
<th class="head"><p>F_beta</p></th>
</tr>
</thead>
<tbody>
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">&quot;micro&quot;</span></code></p></td>
<td><p><span class="math notranslate nohighlight">\(P(y, \hat{y})\)</span></p></td>
<td><p><span class="math notranslate nohighlight">\(R(y, \hat{y})\)</span></p></td>
<td><p><span class="math notranslate nohighlight">\(F_\beta(y, \hat{y})\)</span></p></td>
</tr>
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">&quot;samples&quot;</span></code></p></td>
<td><p><span class="math notranslate nohighlight">\(\frac{1}{\left|S\right|} \sum_{s \in S} P(y_s, \hat{y}_s)\)</span></p></td>
<td><p><span class="math notranslate nohighlight">\(\frac{1}{\left|S\right|} \sum_{s \in S} R(y_s, \hat{y}_s)\)</span></p></td>
<td><p><span class="math notranslate nohighlight">\(\frac{1}{\left|S\right|} \sum_{s \in S} F_\beta(y_s, \hat{y}_s)\)</span></p></td>
</tr>
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">&quot;macro&quot;</span></code></p></td>
<td><p><span class="math notranslate nohighlight">\(\frac{1}{\left|L\right|} \sum_{l \in L} P(y_l, \hat{y}_l)\)</span></p></td>
<td><p><span class="math notranslate nohighlight">\(\frac{1}{\left|L\right|} \sum_{l \in L} R(y_l, \hat{y}_l)\)</span></p></td>
<td><p><span class="math notranslate nohighlight">\(\frac{1}{\left|L\right|} \sum_{l \in L} F_\beta(y_l, \hat{y}_l)\)</span></p></td>
</tr>
<tr class="row-odd"><td><p><code class="docutils literal notranslate"><span class="pre">&quot;weighted&quot;</span></code></p></td>
<td><p><span class="math notranslate nohighlight">\(\frac{1}{\sum_{l \in L} \left|\hat{y}_l\right|} \sum_{l \in L} \left|\hat{y}_l\right| P(y_l, \hat{y}_l)\)</span></p></td>
<td><p><span class="math notranslate nohighlight">\(\frac{1}{\sum_{l \in L} \left|\hat{y}_l\right|} \sum_{l \in L} \left|\hat{y}_l\right| R(y_l, \hat{y}_l)\)</span></p></td>
<td><p><span class="math notranslate nohighlight">\(\frac{1}{\sum_{l \in L} \left|\hat{y}_l\right|} \sum_{l \in L} \left|\hat{y}_l\right| F_\beta(y_l, \hat{y}_l)\)</span></p></td>
</tr>
<tr class="row-even"><td><p><code class="docutils literal notranslate"><span class="pre">None</span></code></p></td>
<td><p><span class="math notranslate nohighlight">\(\langle P(y_l, \hat{y}_l) | l \in L \rangle\)</span></p></td>
<td><p><span class="math notranslate nohighlight">\(\langle R(y_l, \hat{y}_l) | l \in L \rangle\)</span></p></td>
<td><p><span class="math notranslate nohighlight">\(\langle F_\beta(y_l, \hat{y}_l) | l \in L \rangle\)</span></p></td>
</tr>
</tbody>
</table>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn</span> <span class="kn">import</span> <span class="n">metrics</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_true</span> <span class="o">=</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_pred</span> <span class="o">=</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">metrics</span><span class="o">.</span><span class="n">precision_score</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">,</span> <span class="n">average</span><span class="o">=</span><span class="s1">&#39;macro&#39;</span><span class="p">)</span>
<span class="go">0.22...</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">metrics</span><span class="o">.</span><span class="n">recall_score</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">,</span> <span class="n">average</span><span class="o">=</span><span class="s1">&#39;micro&#39;</span><span class="p">)</span>
<span class="go">0.33...</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">metrics</span><span class="o">.</span><span class="n">f1_score</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">,</span> <span class="n">average</span><span class="o">=</span><span class="s1">&#39;weighted&#39;</span><span class="p">)</span>
<span class="go">0.26...</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">metrics</span><span class="o">.</span><span class="n">fbeta_score</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">,</span> <span class="n">average</span><span class="o">=</span><span class="s1">&#39;macro&#39;</span><span class="p">,</span> <span class="n">beta</span><span class="o">=</span><span class="mf">0.5</span><span class="p">)</span>
<span class="go">0.23...</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">metrics</span><span class="o">.</span><span class="n">precision_recall_fscore_support</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">,</span> <span class="n">beta</span><span class="o">=</span><span class="mf">0.5</span><span class="p">,</span> <span class="n">average</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
<span class="go">(array([0.66..., 0.        , 0.        ]), array([1., 0., 0.]), array([0.71..., 0.        , 0.        ]), array([2, 2, 2]...))</span>
</pre></div>
</div>
<p>For multiclass classification with a “negative class”, it is possible to exclude some labels:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">metrics</span><span class="o">.</span><span class="n">recall_score</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">,</span> <span class="n">labels</span><span class="o">=</span><span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="n">average</span><span class="o">=</span><span class="s1">&#39;micro&#39;</span><span class="p">)</span>
<span class="gp">... </span><span class="c1"># excluding 0, no labels were correctly recalled</span>
<span class="go">0.0</span>
</pre></div>
</div>
<p>Similarly, labels not present in the data sample may be accounted for in macro-averaging.</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">metrics</span><span class="o">.</span><span class="n">precision_score</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">,</span> <span class="n">labels</span><span class="o">=</span><span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">],</span> <span class="n">average</span><span class="o">=</span><span class="s1">&#39;macro&#39;</span><span class="p">)</span>
<span class="go">0.166...</span>
</pre></div>
</div>
</div>
</div>
<div class="section" id="jaccard-similarity-coefficient-score">
<span id="jaccard-similarity-score"></span><h3>3.3.2.9. Jaccard similarity coefficient score<a class="headerlink" href="#jaccard-similarity-coefficient-score" title="Permalink to this headline">¶</a></h3>
<p>The <a class="reference internal" href="generated/sklearn.metrics.jaccard_score.html#sklearn.metrics.jaccard_score" title="sklearn.metrics.jaccard_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">jaccard_score</span></code></a> function computes the average of <a class="reference external" href="https://en.wikipedia.org/wiki/Jaccard_index">Jaccard similarity
coefficients</a>, also called the
Jaccard index, between pairs of label sets.</p>
<p>The Jaccard similarity coefficient of the <span class="math notranslate nohighlight">\(i\)</span>-th samples,
with a ground truth label set <span class="math notranslate nohighlight">\(y_i\)</span> and predicted label set
<span class="math notranslate nohighlight">\(\hat{y}_i\)</span>, is defined as</p>
<div class="math notranslate nohighlight">
\[J(y_i, \hat{y}_i) = \frac{|y_i \cap \hat{y}_i|}{|y_i \cup \hat{y}_i|}.\]</div>
<p><a class="reference internal" href="generated/sklearn.metrics.jaccard_score.html#sklearn.metrics.jaccard_score" title="sklearn.metrics.jaccard_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">jaccard_score</span></code></a> works like <a class="reference internal" href="generated/sklearn.metrics.precision_recall_fscore_support.html#sklearn.metrics.precision_recall_fscore_support" title="sklearn.metrics.precision_recall_fscore_support"><code class="xref py py-func docutils literal notranslate"><span class="pre">precision_recall_fscore_support</span></code></a> as a
naively set-wise measure applying natively to binary targets, and extended to
apply to multilabel and multiclass through the use of <code class="docutils literal notranslate"><span class="pre">average</span></code> (see
<a class="reference internal" href="#average"><span class="std std-ref">above</span></a>).</p>
<p>In the binary case:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="kn">import</span> <span class="n">jaccard_score</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_true</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span>
<span class="gp">... </span>                   <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_pred</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span>
<span class="gp">... </span>                   <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">jaccard_score</span><span class="p">(</span><span class="n">y_true</span><span class="p">[</span><span class="mi">0</span><span class="p">],</span> <span class="n">y_pred</span><span class="p">[</span><span class="mi">0</span><span class="p">])</span>
<span class="go">0.6666...</span>
</pre></div>
</div>
<p>In the multilabel case with binary label indicators:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">jaccard_score</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">,</span> <span class="n">average</span><span class="o">=</span><span class="s1">&#39;samples&#39;</span><span class="p">)</span>
<span class="go">0.5833...</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">jaccard_score</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">,</span> <span class="n">average</span><span class="o">=</span><span class="s1">&#39;macro&#39;</span><span class="p">)</span>
<span class="go">0.6666...</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">jaccard_score</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">,</span> <span class="n">average</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
<span class="go">array([0.5, 0.5, 1. ])</span>
</pre></div>
</div>
<p>Multiclass problems are binarized and treated like the corresponding
multilabel problem:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">y_pred</span> <span class="o">=</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_true</span> <span class="o">=</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">jaccard_score</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">,</span> <span class="n">average</span><span class="o">=</span><span class="kc">None</span><span class="p">)</span>
<span class="go">array([1. , 0. , 0.33...])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">jaccard_score</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">,</span> <span class="n">average</span><span class="o">=</span><span class="s1">&#39;macro&#39;</span><span class="p">)</span>
<span class="go">0.44...</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">jaccard_score</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">,</span> <span class="n">average</span><span class="o">=</span><span class="s1">&#39;micro&#39;</span><span class="p">)</span>
<span class="go">0.33...</span>
</pre></div>
</div>
</div>
<div class="section" id="hinge-loss">
<span id="id20"></span><h3>3.3.2.10. Hinge loss<a class="headerlink" href="#hinge-loss" title="Permalink to this headline">¶</a></h3>
<p>The <a class="reference internal" href="generated/sklearn.metrics.hinge_loss.html#sklearn.metrics.hinge_loss" title="sklearn.metrics.hinge_loss"><code class="xref py py-func docutils literal notranslate"><span class="pre">hinge_loss</span></code></a> function computes the average distance between
the model and the data using
<a class="reference external" href="https://en.wikipedia.org/wiki/Hinge_loss">hinge loss</a>, a one-sided metric
that considers only prediction errors. (Hinge
loss is used in maximal margin classifiers such as support vector machines.)</p>
<p>If the labels are encoded with +1 and -1,  <span class="math notranslate nohighlight">\(y\)</span>: is the true
value, and <span class="math notranslate nohighlight">\(w\)</span> is the predicted decisions as output by
<code class="docutils literal notranslate"><span class="pre">decision_function</span></code>, then the hinge loss is defined as:</p>
<div class="math notranslate nohighlight">
\[L_\text{Hinge}(y, w) = \max\left\{1 - wy, 0\right\} = \left|1 - wy\right|_+\]</div>
<p>If there are more than two labels, <a class="reference internal" href="generated/sklearn.metrics.hinge_loss.html#sklearn.metrics.hinge_loss" title="sklearn.metrics.hinge_loss"><code class="xref py py-func docutils literal notranslate"><span class="pre">hinge_loss</span></code></a> uses a multiclass variant
due to Crammer &amp; Singer.
<a class="reference external" href="http://jmlr.csail.mit.edu/papers/volume2/crammer01a/crammer01a.pdf">Here</a> is
the paper describing it.</p>
<p>If <span class="math notranslate nohighlight">\(y_w\)</span> is the predicted decision for true label and <span class="math notranslate nohighlight">\(y_t\)</span> is the
maximum of the predicted decisions for all other labels, where predicted
decisions are output by decision function, then multiclass hinge loss is defined
by:</p>
<div class="math notranslate nohighlight">
\[L_\text{Hinge}(y_w, y_t) = \max\left\{1 + y_t - y_w, 0\right\}\]</div>
<p>Here a small example demonstrating the use of the <a class="reference internal" href="generated/sklearn.metrics.hinge_loss.html#sklearn.metrics.hinge_loss" title="sklearn.metrics.hinge_loss"><code class="xref py py-func docutils literal notranslate"><span class="pre">hinge_loss</span></code></a> function
with a svm classifier in a binary class problem:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn</span> <span class="kn">import</span> <span class="n">svm</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="kn">import</span> <span class="n">hinge_loss</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">X</span> <span class="o">=</span> <span class="p">[[</span><span class="mi">0</span><span class="p">],</span> <span class="p">[</span><span class="mi">1</span><span class="p">]]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y</span> <span class="o">=</span> <span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">est</span> <span class="o">=</span> <span class="n">svm</span><span class="o">.</span><span class="n">LinearSVC</span><span class="p">(</span><span class="n">random_state</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">est</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">)</span>
<span class="go">LinearSVC(random_state=0)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">pred_decision</span> <span class="o">=</span> <span class="n">est</span><span class="o">.</span><span class="n">decision_function</span><span class="p">([[</span><span class="o">-</span><span class="mi">2</span><span class="p">],</span> <span class="p">[</span><span class="mi">3</span><span class="p">],</span> <span class="p">[</span><span class="mf">0.5</span><span class="p">]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">pred_decision</span>
<span class="go">array([-2.18...,  2.36...,  0.09...])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">hinge_loss</span><span class="p">([</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span> <span class="n">pred_decision</span><span class="p">)</span>
<span class="go">0.3...</span>
</pre></div>
</div>
<p>Here is an example demonstrating the use of the <a class="reference internal" href="generated/sklearn.metrics.hinge_loss.html#sklearn.metrics.hinge_loss" title="sklearn.metrics.hinge_loss"><code class="xref py py-func docutils literal notranslate"><span class="pre">hinge_loss</span></code></a> function
with a svm classifier in a multiclass problem:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">X</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([[</span><span class="mi">0</span><span class="p">],</span> <span class="p">[</span><span class="mi">1</span><span class="p">],</span> <span class="p">[</span><span class="mi">2</span><span class="p">],</span> <span class="p">[</span><span class="mi">3</span><span class="p">]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">Y</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">labels</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">est</span> <span class="o">=</span> <span class="n">svm</span><span class="o">.</span><span class="n">LinearSVC</span><span class="p">()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">est</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">Y</span><span class="p">)</span>
<span class="go">LinearSVC()</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">pred_decision</span> <span class="o">=</span> <span class="n">est</span><span class="o">.</span><span class="n">decision_function</span><span class="p">([[</span><span class="o">-</span><span class="mi">1</span><span class="p">],</span> <span class="p">[</span><span class="mi">2</span><span class="p">],</span> <span class="p">[</span><span class="mi">3</span><span class="p">]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_true</span> <span class="o">=</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">hinge_loss</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">pred_decision</span><span class="p">,</span> <span class="n">labels</span><span class="p">)</span>
<span class="go">0.56...</span>
</pre></div>
</div>
</div>
<div class="section" id="log-loss">
<span id="id22"></span><h3>3.3.2.11. Log loss<a class="headerlink" href="#log-loss" title="Permalink to this headline">¶</a></h3>
<p>Log loss, also called logistic regression loss or
cross-entropy loss, is defined on probability estimates.  It is
commonly used in (multinomial) logistic regression and neural networks, as well
as in some variants of expectation-maximization, and can be used to evaluate the
probability outputs (<code class="docutils literal notranslate"><span class="pre">predict_proba</span></code>) of a classifier instead of its
discrete predictions.</p>
<p>For binary classification with a true label <span class="math notranslate nohighlight">\(y \in \{0,1\}\)</span>
and a probability estimate <span class="math notranslate nohighlight">\(p = \operatorname{Pr}(y = 1)\)</span>,
the log loss per sample is the negative log-likelihood
of the classifier given the true label:</p>
<div class="math notranslate nohighlight">
\[L_{\log}(y, p) = -\log \operatorname{Pr}(y|p) = -(y \log (p) + (1 - y) \log (1 - p))\]</div>
<p>This extends to the multiclass case as follows.
Let the true labels for a set of samples
be encoded as a 1-of-K binary indicator matrix <span class="math notranslate nohighlight">\(Y\)</span>,
i.e., <span class="math notranslate nohighlight">\(y_{i,k} = 1\)</span> if sample <span class="math notranslate nohighlight">\(i\)</span> has label <span class="math notranslate nohighlight">\(k\)</span>
taken from a set of <span class="math notranslate nohighlight">\(K\)</span> labels.
Let <span class="math notranslate nohighlight">\(P\)</span> be a matrix of probability estimates,
with <span class="math notranslate nohighlight">\(p_{i,k} = \operatorname{Pr}(t_{i,k} = 1)\)</span>.
Then the log loss of the whole set is</p>
<div class="math notranslate nohighlight">
\[L_{\log}(Y, P) = -\log \operatorname{Pr}(Y|P) = - \frac{1}{N} \sum_{i=0}^{N-1} \sum_{k=0}^{K-1} y_{i,k} \log p_{i,k}\]</div>
<p>To see how this generalizes the binary log loss given above,
note that in the binary case,
<span class="math notranslate nohighlight">\(p_{i,0} = 1 - p_{i,1}\)</span> and <span class="math notranslate nohighlight">\(y_{i,0} = 1 - y_{i,1}\)</span>,
so expanding the inner sum over <span class="math notranslate nohighlight">\(y_{i,k} \in \{0,1\}\)</span>
gives the binary log loss.</p>
<p>The <a class="reference internal" href="generated/sklearn.metrics.log_loss.html#sklearn.metrics.log_loss" title="sklearn.metrics.log_loss"><code class="xref py py-func docutils literal notranslate"><span class="pre">log_loss</span></code></a> function computes log loss given a list of ground-truth
labels and a probability matrix, as returned by an estimator’s <code class="docutils literal notranslate"><span class="pre">predict_proba</span></code>
method.</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="kn">import</span> <span class="n">log_loss</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_true</span> <span class="o">=</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_pred</span> <span class="o">=</span> <span class="p">[[</span><span class="o">.</span><span class="mi">9</span><span class="p">,</span> <span class="o">.</span><span class="mi">1</span><span class="p">],</span> <span class="p">[</span><span class="o">.</span><span class="mi">8</span><span class="p">,</span> <span class="o">.</span><span class="mi">2</span><span class="p">],</span> <span class="p">[</span><span class="o">.</span><span class="mi">3</span><span class="p">,</span> <span class="o">.</span><span class="mi">7</span><span class="p">],</span> <span class="p">[</span><span class="o">.</span><span class="mi">01</span><span class="p">,</span> <span class="o">.</span><span class="mi">99</span><span class="p">]]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">log_loss</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">)</span>
<span class="go">0.1738...</span>
</pre></div>
</div>
<p>The first <code class="docutils literal notranslate"><span class="pre">[.9,</span> <span class="pre">.1]</span></code> in <code class="docutils literal notranslate"><span class="pre">y_pred</span></code> denotes 90% probability that the first
sample has label 0.  The log loss is non-negative.</p>
</div>
<div class="section" id="matthews-correlation-coefficient">
<span id="matthews-corrcoef"></span><h3>3.3.2.12. Matthews correlation coefficient<a class="headerlink" href="#matthews-correlation-coefficient" title="Permalink to this headline">¶</a></h3>
<p>The <a class="reference internal" href="generated/sklearn.metrics.matthews_corrcoef.html#sklearn.metrics.matthews_corrcoef" title="sklearn.metrics.matthews_corrcoef"><code class="xref py py-func docutils literal notranslate"><span class="pre">matthews_corrcoef</span></code></a> function computes the
<a class="reference external" href="https://en.wikipedia.org/wiki/Matthews_correlation_coefficient">Matthew’s correlation coefficient (MCC)</a>
for binary classes.  Quoting Wikipedia:</p>
<blockquote>
<div><p>“The Matthews correlation coefficient is used in machine learning as a
measure of the quality of binary (two-class) classifications. It takes
into account true and false positives and negatives and is generally
regarded as a balanced measure which can be used even if the classes are
of very different sizes. The MCC is in essence a correlation coefficient
value between -1 and +1. A coefficient of +1 represents a perfect
prediction, 0 an average random prediction and -1 an inverse prediction.
The statistic is also known as the phi coefficient.”</p>
</div></blockquote>
<p>In the binary (two-class) case, <span class="math notranslate nohighlight">\(tp\)</span>, <span class="math notranslate nohighlight">\(tn\)</span>, <span class="math notranslate nohighlight">\(fp\)</span> and
<span class="math notranslate nohighlight">\(fn\)</span> are respectively the number of true positives, true negatives, false
positives and false negatives, the MCC is defined as</p>
<div class="math notranslate nohighlight">
\[MCC = \frac{tp \times tn - fp \times fn}{\sqrt{(tp + fp)(tp + fn)(tn + fp)(tn + fn)}}.\]</div>
<p>In the multiclass case, the Matthews correlation coefficient can be <a class="reference external" href="http://rk.kvl.dk/introduction/index.html">defined</a> in terms of a
<a class="reference internal" href="generated/sklearn.metrics.confusion_matrix.html#sklearn.metrics.confusion_matrix" title="sklearn.metrics.confusion_matrix"><code class="xref py py-func docutils literal notranslate"><span class="pre">confusion_matrix</span></code></a> <span class="math notranslate nohighlight">\(C\)</span> for <span class="math notranslate nohighlight">\(K\)</span> classes.  To simplify the
definition consider the following intermediate variables:</p>
<ul class="simple">
<li><p><span class="math notranslate nohighlight">\(t_k=\sum_{i}^{K} C_{ik}\)</span> the number of times class <span class="math notranslate nohighlight">\(k\)</span> truly occurred,</p></li>
<li><p><span class="math notranslate nohighlight">\(p_k=\sum_{i}^{K} C_{ki}\)</span> the number of times class <span class="math notranslate nohighlight">\(k\)</span> was predicted,</p></li>
<li><p><span class="math notranslate nohighlight">\(c=\sum_{k}^{K} C_{kk}\)</span> the total number of samples correctly predicted,</p></li>
<li><p><span class="math notranslate nohighlight">\(s=\sum_{i}^{K} \sum_{j}^{K} C_{ij}\)</span> the total number of samples.</p></li>
</ul>
<p>Then the multiclass MCC is defined as:</p>
<div class="math notranslate nohighlight">
\[MCC = \frac{
    c \times s - \sum_{k}^{K} p_k \times t_k
}{\sqrt{
    (s^2 - \sum_{k}^{K} p_k^2) \times
    (s^2 - \sum_{k}^{K} t_k^2)
}}\]</div>
<p>When there are more than two labels, the value of the MCC will no longer range
between -1 and +1. Instead the minimum value will be somewhere between -1 and 0
depending on the number and distribution of ground true labels. The maximum
value is always +1.</p>
<p>Here is a small example illustrating the usage of the <a class="reference internal" href="generated/sklearn.metrics.matthews_corrcoef.html#sklearn.metrics.matthews_corrcoef" title="sklearn.metrics.matthews_corrcoef"><code class="xref py py-func docutils literal notranslate"><span class="pre">matthews_corrcoef</span></code></a>
function:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="kn">import</span> <span class="n">matthews_corrcoef</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_true</span> <span class="o">=</span> <span class="p">[</span><span class="o">+</span><span class="mi">1</span><span class="p">,</span> <span class="o">+</span><span class="mi">1</span><span class="p">,</span> <span class="o">+</span><span class="mi">1</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_pred</span> <span class="o">=</span> <span class="p">[</span><span class="o">+</span><span class="mi">1</span><span class="p">,</span> <span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="o">+</span><span class="mi">1</span><span class="p">,</span> <span class="o">+</span><span class="mi">1</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">matthews_corrcoef</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">)</span>
<span class="go">-0.33...</span>
</pre></div>
</div>
</div>
<div class="section" id="multi-label-confusion-matrix">
<span id="multilabel-confusion-matrix"></span><h3>3.3.2.13. Multi-label confusion matrix<a class="headerlink" href="#multi-label-confusion-matrix" title="Permalink to this headline">¶</a></h3>
<p>The <a class="reference internal" href="generated/sklearn.metrics.multilabel_confusion_matrix.html#sklearn.metrics.multilabel_confusion_matrix" title="sklearn.metrics.multilabel_confusion_matrix"><code class="xref py py-func docutils literal notranslate"><span class="pre">multilabel_confusion_matrix</span></code></a> function computes class-wise (default)
or sample-wise (samplewise=True) multilabel confusion matrix to evaluate
the accuracy of a classification. multilabel_confusion_matrix also treats
multiclass data as if it were multilabel, as this is a transformation commonly
applied to evaluate multiclass problems with binary classification metrics
(such as precision, recall, etc.).</p>
<p>When calculating class-wise multilabel confusion matrix <span class="math notranslate nohighlight">\(C\)</span>, the
count of true negatives for class <span class="math notranslate nohighlight">\(i\)</span> is <span class="math notranslate nohighlight">\(C_{i,0,0}\)</span>, false
negatives is <span class="math notranslate nohighlight">\(C_{i,1,0}\)</span>, true positives is <span class="math notranslate nohighlight">\(C_{i,1,1}\)</span>
and false positives is <span class="math notranslate nohighlight">\(C_{i,0,1}\)</span>.</p>
<p>Here is an example demonstrating the use of the
<a class="reference internal" href="generated/sklearn.metrics.multilabel_confusion_matrix.html#sklearn.metrics.multilabel_confusion_matrix" title="sklearn.metrics.multilabel_confusion_matrix"><code class="xref py py-func docutils literal notranslate"><span class="pre">multilabel_confusion_matrix</span></code></a> function with
<a class="reference internal" href="../glossary.html#term-multilabel-indicator-matrix"><span class="xref std std-term">multilabel indicator matrix</span></a> input:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="kn">import</span> <span class="n">multilabel_confusion_matrix</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_true</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span>
<span class="gp">... </span>                   <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_pred</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">],</span>
<span class="gp">... </span>                   <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">multilabel_confusion_matrix</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">)</span>
<span class="go">array([[[1, 0],</span>
<span class="go">        [0, 1]],</span>

<span class="go">       [[1, 0],</span>
<span class="go">        [0, 1]],</span>

<span class="go">       [[0, 1],</span>
<span class="go">        [1, 0]]])</span>
</pre></div>
</div>
<p>Or a confusion matrix can be constructed for each sample’s labels:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">multilabel_confusion_matrix</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">,</span> <span class="n">samplewise</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="go">array([[[1, 0],</span>
<span class="go">        [1, 1]],</span>
<span class="go">&lt;BLANKLINE&gt;</span>
<span class="go">       [[1, 1],</span>
<span class="go">        [0, 1]]])</span>
</pre></div>
</div>
<p>Here is an example demonstrating the use of the
<a class="reference internal" href="generated/sklearn.metrics.multilabel_confusion_matrix.html#sklearn.metrics.multilabel_confusion_matrix" title="sklearn.metrics.multilabel_confusion_matrix"><code class="xref py py-func docutils literal notranslate"><span class="pre">multilabel_confusion_matrix</span></code></a> function with
<a class="reference internal" href="../glossary.html#term-multiclass"><span class="xref std std-term">multiclass</span></a> input:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">y_true</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;cat&quot;</span><span class="p">,</span> <span class="s2">&quot;ant&quot;</span><span class="p">,</span> <span class="s2">&quot;cat&quot;</span><span class="p">,</span> <span class="s2">&quot;cat&quot;</span><span class="p">,</span> <span class="s2">&quot;ant&quot;</span><span class="p">,</span> <span class="s2">&quot;bird&quot;</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_pred</span> <span class="o">=</span> <span class="p">[</span><span class="s2">&quot;ant&quot;</span><span class="p">,</span> <span class="s2">&quot;ant&quot;</span><span class="p">,</span> <span class="s2">&quot;cat&quot;</span><span class="p">,</span> <span class="s2">&quot;cat&quot;</span><span class="p">,</span> <span class="s2">&quot;ant&quot;</span><span class="p">,</span> <span class="s2">&quot;cat&quot;</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">multilabel_confusion_matrix</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">,</span>
<span class="gp">... </span>                            <span class="n">labels</span><span class="o">=</span><span class="p">[</span><span class="s2">&quot;ant&quot;</span><span class="p">,</span> <span class="s2">&quot;bird&quot;</span><span class="p">,</span> <span class="s2">&quot;cat&quot;</span><span class="p">])</span>
<span class="go">array([[[3, 1],</span>
<span class="go">        [0, 2]],</span>

<span class="go">       [[5, 0],</span>
<span class="go">        [1, 0]],</span>

<span class="go">       [[2, 1],</span>
<span class="go">        [1, 2]]])</span>
</pre></div>
</div>
<p>Here are some examples demonstrating the use of the
<a class="reference internal" href="generated/sklearn.metrics.multilabel_confusion_matrix.html#sklearn.metrics.multilabel_confusion_matrix" title="sklearn.metrics.multilabel_confusion_matrix"><code class="xref py py-func docutils literal notranslate"><span class="pre">multilabel_confusion_matrix</span></code></a> function to calculate recall
(or sensitivity), specificity, fall out and miss rate for each class in a
problem with multilabel indicator matrix input.</p>
<p>Calculating
<a class="reference external" href="https://en.wikipedia.org/wiki/Sensitivity_and_specificity">recall</a>
(also called the true positive rate or the sensitivity) for each class:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">y_true</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span>
<span class="gp">... </span>                   <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">],</span>
<span class="gp">... </span>                   <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_pred</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">],</span>
<span class="gp">... </span>                   <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span>
<span class="gp">... </span>                   <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">mcm</span> <span class="o">=</span> <span class="n">multilabel_confusion_matrix</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">tn</span> <span class="o">=</span> <span class="n">mcm</span><span class="p">[:,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">tp</span> <span class="o">=</span> <span class="n">mcm</span><span class="p">[:,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">fn</span> <span class="o">=</span> <span class="n">mcm</span><span class="p">[:,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">fp</span> <span class="o">=</span> <span class="n">mcm</span><span class="p">[:,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">tp</span> <span class="o">/</span> <span class="p">(</span><span class="n">tp</span> <span class="o">+</span> <span class="n">fn</span><span class="p">)</span>
<span class="go">array([1. , 0.5, 0. ])</span>
</pre></div>
</div>
<p>Calculating
<a class="reference external" href="https://en.wikipedia.org/wiki/Sensitivity_and_specificity">specificity</a>
(also called the true negative rate) for each class:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">tn</span> <span class="o">/</span> <span class="p">(</span><span class="n">tn</span> <span class="o">+</span> <span class="n">fp</span><span class="p">)</span>
<span class="go">array([1. , 0. , 0.5])</span>
</pre></div>
</div>
<p>Calculating <a class="reference external" href="https://en.wikipedia.org/wiki/False_positive_rate">fall out</a>
(also called the false positive rate) for each class:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">fp</span> <span class="o">/</span> <span class="p">(</span><span class="n">fp</span> <span class="o">+</span> <span class="n">tn</span><span class="p">)</span>
<span class="go">array([0. , 1. , 0.5])</span>
</pre></div>
</div>
<p>Calculating <a class="reference external" href="https://en.wikipedia.org/wiki/False_positives_and_false_negatives">miss rate</a>
(also called the false negative rate) for each class:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">fn</span> <span class="o">/</span> <span class="p">(</span><span class="n">fn</span> <span class="o">+</span> <span class="n">tp</span><span class="p">)</span>
<span class="go">array([0. , 0.5, 1. ])</span>
</pre></div>
</div>
</div>
<div class="section" id="receiver-operating-characteristic-roc">
<span id="roc-metrics"></span><h3>3.3.2.14. Receiver operating characteristic (ROC)<a class="headerlink" href="#receiver-operating-characteristic-roc" title="Permalink to this headline">¶</a></h3>
<p>The function <a class="reference internal" href="generated/sklearn.metrics.roc_curve.html#sklearn.metrics.roc_curve" title="sklearn.metrics.roc_curve"><code class="xref py py-func docutils literal notranslate"><span class="pre">roc_curve</span></code></a> computes the
<a class="reference external" href="https://en.wikipedia.org/wiki/Receiver_operating_characteristic">receiver operating characteristic curve, or ROC curve</a>.
Quoting Wikipedia :</p>
<blockquote>
<div><p>“A receiver operating characteristic (ROC), or simply ROC curve, is a
graphical plot which illustrates the performance of a binary classifier
system as its discrimination threshold is varied. It is created by plotting
the fraction of true positives out of the positives (TPR = true positive
rate) vs. the fraction of false positives out of the negatives (FPR = false
positive rate), at various threshold settings. TPR is also known as
sensitivity, and FPR is one minus the specificity or true negative rate.”</p>
</div></blockquote>
<p>This function requires the true binary
value and the target scores, which can either be probability estimates of the
positive class, confidence values, or binary decisions.
Here is a small example of how to use the <a class="reference internal" href="generated/sklearn.metrics.roc_curve.html#sklearn.metrics.roc_curve" title="sklearn.metrics.roc_curve"><code class="xref py py-func docutils literal notranslate"><span class="pre">roc_curve</span></code></a> function:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="kn">import</span> <span class="n">roc_curve</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">scores</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mf">0.1</span><span class="p">,</span> <span class="mf">0.4</span><span class="p">,</span> <span class="mf">0.35</span><span class="p">,</span> <span class="mf">0.8</span><span class="p">])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">fpr</span><span class="p">,</span> <span class="n">tpr</span><span class="p">,</span> <span class="n">thresholds</span> <span class="o">=</span> <span class="n">roc_curve</span><span class="p">(</span><span class="n">y</span><span class="p">,</span> <span class="n">scores</span><span class="p">,</span> <span class="n">pos_label</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">fpr</span>
<span class="go">array([0. , 0. , 0.5, 0.5, 1. ])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">tpr</span>
<span class="go">array([0. , 0.5, 0.5, 1. , 1. ])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">thresholds</span>
<span class="go">array([1.8 , 0.8 , 0.4 , 0.35, 0.1 ])</span>
</pre></div>
</div>
<p>This figure shows an example of such an ROC curve:</p>
<a class="reference external image-reference" href="../auto_examples/model_selection/plot_roc.html"><img alt="modules/../auto_examples/model_selection/images/sphx_glr_plot_roc_001.png" class="align-center" src="modules/../auto_examples/model_selection/images/sphx_glr_plot_roc_001.png" /></a>
<p>The <a class="reference internal" href="generated/sklearn.metrics.roc_auc_score.html#sklearn.metrics.roc_auc_score" title="sklearn.metrics.roc_auc_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">roc_auc_score</span></code></a> function computes the area under the receiver
operating characteristic (ROC) curve, which is also denoted by
AUC or AUROC.  By computing the
area under the roc curve, the curve information is summarized in one number.
For more information see the <a class="reference external" href="https://en.wikipedia.org/wiki/Receiver_operating_characteristic#Area_under_the_curve">Wikipedia article on AUC</a>.</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="kn">import</span> <span class="n">roc_auc_score</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_true</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_scores</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mf">0.1</span><span class="p">,</span> <span class="mf">0.4</span><span class="p">,</span> <span class="mf">0.35</span><span class="p">,</span> <span class="mf">0.8</span><span class="p">])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">roc_auc_score</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_scores</span><span class="p">)</span>
<span class="go">0.75</span>
</pre></div>
</div>
<p>In multi-label classification, the <a class="reference internal" href="generated/sklearn.metrics.roc_auc_score.html#sklearn.metrics.roc_auc_score" title="sklearn.metrics.roc_auc_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">roc_auc_score</span></code></a> function is
extended by averaging over the labels as <a class="reference internal" href="#average"><span class="std std-ref">above</span></a>.</p>
<p>Compared to metrics such as the subset accuracy, the Hamming loss, or the
F1 score, ROC doesn’t require optimizing a threshold for each label.</p>
<p>The <a class="reference internal" href="generated/sklearn.metrics.roc_auc_score.html#sklearn.metrics.roc_auc_score" title="sklearn.metrics.roc_auc_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">roc_auc_score</span></code></a> function can also be used in multi-class
classification. Two averaging strategies are currently supported: the
one-vs-one algorithm computes the average of the pairwise ROC AUC scores, and
the one-vs-rest algorithm computes the average of the ROC AUC scores for each
class against all other classes. In both cases, the predicted labels are
provided in an array with values from 0 to <code class="docutils literal notranslate"><span class="pre">n_classes</span></code>, and the scores
correspond to the probability estimates that a sample belongs to a particular
class. The OvO and OvR algorithms supports weighting uniformly
(<code class="docutils literal notranslate"><span class="pre">average='macro'</span></code>) and weighting by the prevalence (<code class="docutils literal notranslate"><span class="pre">average='weighted'</span></code>).</p>
<p><strong>One-vs-one Algorithm</strong>: Computes the average AUC of all possible pairwise
combinations of classes. <a class="reference internal" href="#ht2001" id="id23"><span>[HT2001]</span></a> defines a multiclass AUC metric weighted
uniformly:</p>
<div class="math notranslate nohighlight">
\[\frac{2}{c(c-1)}\sum_{j=1}^{c}\sum_{k &gt; j}^c (\text{AUC}(j | k) +
\text{AUC}(k | j))\]</div>
<p>where <span class="math notranslate nohighlight">\(c\)</span> is the number of classes and <span class="math notranslate nohighlight">\(\text{AUC}(j | k)\)</span> is the
AUC with class <span class="math notranslate nohighlight">\(j\)</span> as the positive class and class <span class="math notranslate nohighlight">\(k\)</span> as the
negative class. In general,
<span class="math notranslate nohighlight">\(\text{AUC}(j | k) \neq \text{AUC}(k | j))\)</span> in the multiclass
case. This algorithm is used by setting the keyword argument <code class="docutils literal notranslate"><span class="pre">multiclass</span></code>
to <code class="docutils literal notranslate"><span class="pre">'ovo'</span></code> and <code class="docutils literal notranslate"><span class="pre">average</span></code> to <code class="docutils literal notranslate"><span class="pre">'macro'</span></code>.</p>
<p>The <a class="reference internal" href="#ht2001" id="id24"><span>[HT2001]</span></a> multiclass AUC metric can be extended to be weighted by the
prevalence:</p>
<div class="math notranslate nohighlight">
\[\frac{2}{c(c-1)}\sum_{j=1}^{c}\sum_{k &gt; j}^c p(j \cup k)(
\text{AUC}(j | k) + \text{AUC}(k | j))\]</div>
<p>where <span class="math notranslate nohighlight">\(c\)</span> is the number of classes. This algorithm is used by setting
the keyword argument <code class="docutils literal notranslate"><span class="pre">multiclass</span></code> to <code class="docutils literal notranslate"><span class="pre">'ovo'</span></code> and <code class="docutils literal notranslate"><span class="pre">average</span></code> to
<code class="docutils literal notranslate"><span class="pre">'weighted'</span></code>. The <code class="docutils literal notranslate"><span class="pre">'weighted'</span></code> option returns a prevalence-weighted average
as described in <a class="reference internal" href="#fc2009" id="id25"><span>[FC2009]</span></a>.</p>
<p><strong>One-vs-rest Algorithm</strong>: Computes the AUC of each class against the rest.
The algorithm is functionally the same as the multilabel case. To enable this
algorithm set the keyword argument <code class="docutils literal notranslate"><span class="pre">multiclass</span></code> to <code class="docutils literal notranslate"><span class="pre">'ovr'</span></code>. Similar to
OvO, OvR supports two types of averaging: <code class="docutils literal notranslate"><span class="pre">'macro'</span></code> <a class="reference internal" href="#f2006" id="id26"><span>[F2006]</span></a> and
<code class="docutils literal notranslate"><span class="pre">'weighted'</span></code> <a class="reference internal" href="#f2001" id="id27"><span>[F2001]</span></a>.</p>
<p>In applications where a high false positive rate is not tolerable the parameter
<code class="docutils literal notranslate"><span class="pre">max_fpr</span></code> of <a class="reference internal" href="generated/sklearn.metrics.roc_auc_score.html#sklearn.metrics.roc_auc_score" title="sklearn.metrics.roc_auc_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">roc_auc_score</span></code></a> can be used to summarize the ROC curve up
to the given limit.</p>
<a class="reference external image-reference" href="../auto_examples/model_selection/plot_roc.html"><img alt="modules/../auto_examples/model_selection/images/sphx_glr_plot_roc_002.png" class="align-center" src="modules/../auto_examples/model_selection/images/sphx_glr_plot_roc_002.png" /></a>
<div class="topic">
<p class="topic-title">Examples:</p>
<ul class="simple">
<li><p>See <a class="reference internal" href="../auto_examples/model_selection/plot_roc.html#sphx-glr-auto-examples-model-selection-plot-roc-py"><span class="std std-ref">Receiver Operating Characteristic (ROC)</span></a>
for an example of using ROC to
evaluate the quality of the output of a classifier.</p></li>
<li><p>See <a class="reference internal" href="../auto_examples/model_selection/plot_roc_crossval.html#sphx-glr-auto-examples-model-selection-plot-roc-crossval-py"><span class="std std-ref">Receiver Operating Characteristic (ROC) with cross validation</span></a>
for an example of using ROC to
evaluate classifier output quality, using cross-validation.</p></li>
<li><p>See <a class="reference internal" href="../auto_examples/applications/plot_species_distribution_modeling.html#sphx-glr-auto-examples-applications-plot-species-distribution-modeling-py"><span class="std std-ref">Species distribution modeling</span></a>
for an example of using ROC to
model species distribution.</p></li>
</ul>
</div>
<div class="topic">
<p class="topic-title">References:</p>
<dl class="citation">
<dt class="label" id="ht2001"><span class="brackets">HT2001</span><span class="fn-backref">(<a href="#id23">1</a>,<a href="#id24">2</a>)</span></dt>
<dd><p>Hand, D.J. and Till, R.J., (2001). <a class="reference external" href="http://link.springer.com/article/10.1023/A:1010920819831">A simple generalisation
of the area under the ROC curve for multiple class classification problems.</a>
Machine learning, 45(2), pp.171-186.</p>
</dd>
<dt class="label" id="fc2009"><span class="brackets"><a class="fn-backref" href="#id25">FC2009</a></span></dt>
<dd><p>Ferri, Cèsar &amp; Hernandez-Orallo, Jose &amp; Modroiu, R. (2009).
<a class="reference external" href="https://www.math.ucdavis.edu/~saito/data/roc/ferri-class-perf-metrics.pdf">An Experimental Comparison of Performance Measures for Classification.</a>
Pattern Recognition Letters. 30. 27-38.</p>
</dd>
<dt class="label" id="f2006"><span class="brackets"><a class="fn-backref" href="#id26">F2006</a></span></dt>
<dd><p>Fawcett, T., 2006. <a class="reference external" href="http://www.sciencedirect.com/science/article/pii/S016786550500303X">An introduction to ROC analysis.</a>
Pattern Recognition Letters, 27(8), pp. 861-874.</p>
</dd>
<dt class="label" id="f2001"><span class="brackets"><a class="fn-backref" href="#id27">F2001</a></span></dt>
<dd><p>Fawcett, T., 2001. <a class="reference external" href="http://ieeexplore.ieee.org/document/989510/">Using rule sets to maximize
ROC performance</a>
In Data Mining, 2001.
Proceedings IEEE International Conference, pp. 131-138.</p>
</dd>
</dl>
</div>
</div>
<div class="section" id="zero-one-loss">
<span id="id28"></span><h3>3.3.2.15. Zero one loss<a class="headerlink" href="#zero-one-loss" title="Permalink to this headline">¶</a></h3>
<p>The <a class="reference internal" href="generated/sklearn.metrics.zero_one_loss.html#sklearn.metrics.zero_one_loss" title="sklearn.metrics.zero_one_loss"><code class="xref py py-func docutils literal notranslate"><span class="pre">zero_one_loss</span></code></a> function computes the sum or the average of the 0-1
classification loss (<span class="math notranslate nohighlight">\(L_{0-1}\)</span>) over <span class="math notranslate nohighlight">\(n_{\text{samples}}\)</span>. By
default, the function normalizes over the sample. To get the sum of the
<span class="math notranslate nohighlight">\(L_{0-1}\)</span>, set <code class="docutils literal notranslate"><span class="pre">normalize</span></code> to <code class="docutils literal notranslate"><span class="pre">False</span></code>.</p>
<p>In multilabel classification, the <a class="reference internal" href="generated/sklearn.metrics.zero_one_loss.html#sklearn.metrics.zero_one_loss" title="sklearn.metrics.zero_one_loss"><code class="xref py py-func docutils literal notranslate"><span class="pre">zero_one_loss</span></code></a> scores a subset as
one if its labels strictly match the predictions, and as a zero if there
are any errors.  By default, the function returns the percentage of imperfectly
predicted subsets.  To get the count of such subsets instead, set
<code class="docutils literal notranslate"><span class="pre">normalize</span></code> to <code class="docutils literal notranslate"><span class="pre">False</span></code></p>
<p>If <span class="math notranslate nohighlight">\(\hat{y}_i\)</span> is the predicted value of
the <span class="math notranslate nohighlight">\(i\)</span>-th sample and <span class="math notranslate nohighlight">\(y_i\)</span> is the corresponding true value,
then the 0-1 loss <span class="math notranslate nohighlight">\(L_{0-1}\)</span> is defined as:</p>
<div class="math notranslate nohighlight">
\[L_{0-1}(y_i, \hat{y}_i) = 1(\hat{y}_i \not= y_i)\]</div>
<p>where <span class="math notranslate nohighlight">\(1(x)\)</span> is the <a class="reference external" href="https://en.wikipedia.org/wiki/Indicator_function">indicator function</a>.</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="kn">import</span> <span class="n">zero_one_loss</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_pred</span> <span class="o">=</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_true</span> <span class="o">=</span> <span class="p">[</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">3</span><span class="p">,</span> <span class="mi">4</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">zero_one_loss</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">)</span>
<span class="go">0.25</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">zero_one_loss</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">,</span> <span class="n">normalize</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
<span class="go">1</span>
</pre></div>
</div>
<p>In the multilabel case with binary label indicators, where the first label
set [0,1] has an error:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">zero_one_loss</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">]]),</span> <span class="n">np</span><span class="o">.</span><span class="n">ones</span><span class="p">((</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">)))</span>
<span class="go">0.5</span>

<span class="gp">&gt;&gt;&gt; </span><span class="n">zero_one_loss</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">]]),</span> <span class="n">np</span><span class="o">.</span><span class="n">ones</span><span class="p">((</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">)),</span>  <span class="n">normalize</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
<span class="go">1</span>
</pre></div>
</div>
<div class="topic">
<p class="topic-title">Example:</p>
<ul class="simple">
<li><p>See <a class="reference internal" href="../auto_examples/feature_selection/plot_rfe_with_cross_validation.html#sphx-glr-auto-examples-feature-selection-plot-rfe-with-cross-validation-py"><span class="std std-ref">Recursive feature elimination with cross-validation</span></a>
for an example of zero one loss usage to perform recursive feature
elimination with cross-validation.</p></li>
</ul>
</div>
</div>
<div class="section" id="brier-score-loss">
<span id="id30"></span><h3>3.3.2.16. Brier score loss<a class="headerlink" href="#brier-score-loss" title="Permalink to this headline">¶</a></h3>
<p>The <a class="reference internal" href="generated/sklearn.metrics.brier_score_loss.html#sklearn.metrics.brier_score_loss" title="sklearn.metrics.brier_score_loss"><code class="xref py py-func docutils literal notranslate"><span class="pre">brier_score_loss</span></code></a> function computes the
<a class="reference external" href="https://en.wikipedia.org/wiki/Brier_score">Brier score</a>
for binary classes. Quoting Wikipedia:</p>
<blockquote>
<div><p>“The Brier score is a proper score function that measures the accuracy of
probabilistic predictions. It is applicable to tasks in which predictions
must assign probabilities to a set of mutually exclusive discrete outcomes.”</p>
</div></blockquote>
<p>This function returns a score of the mean square difference between the actual
outcome and the predicted probability of the possible outcome. The actual
outcome has to be 1 or 0 (true or false), while the predicted probability of
the actual outcome can be a value between 0 and 1.</p>
<p>The brier score loss is also between 0 to 1 and the lower the score (the mean
square difference is smaller), the more accurate the prediction is. It can be
thought of as a measure of the “calibration” of a set of probabilistic
predictions.</p>
<div class="math notranslate nohighlight">
\[BS = \frac{1}{N} \sum_{t=1}^{N}(f_t - o_t)^2\]</div>
<p>where : <span class="math notranslate nohighlight">\(N\)</span> is the total number of predictions, <span class="math notranslate nohighlight">\(f_t\)</span> is the
predicted probability of the actual outcome <span class="math notranslate nohighlight">\(o_t\)</span>.</p>
<p>Here is a small example of usage of this function::</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="kn">import</span> <span class="n">brier_score_loss</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_true</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_true_categorical</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="s2">&quot;spam&quot;</span><span class="p">,</span> <span class="s2">&quot;ham&quot;</span><span class="p">,</span> <span class="s2">&quot;ham&quot;</span><span class="p">,</span> <span class="s2">&quot;spam&quot;</span><span class="p">])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_prob</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mf">0.1</span><span class="p">,</span> <span class="mf">0.9</span><span class="p">,</span> <span class="mf">0.8</span><span class="p">,</span> <span class="mf">0.4</span><span class="p">])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_pred</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([</span><span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">brier_score_loss</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_prob</span><span class="p">)</span>
<span class="go">0.055</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">brier_score_loss</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="mi">1</span> <span class="o">-</span> <span class="n">y_prob</span><span class="p">,</span> <span class="n">pos_label</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
<span class="go">0.055</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">brier_score_loss</span><span class="p">(</span><span class="n">y_true_categorical</span><span class="p">,</span> <span class="n">y_prob</span><span class="p">,</span> <span class="n">pos_label</span><span class="o">=</span><span class="s2">&quot;ham&quot;</span><span class="p">)</span>
<span class="go">0.055</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">brier_score_loss</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_prob</span> <span class="o">&gt;</span> <span class="mf">0.5</span><span class="p">)</span>
<span class="go">0.0</span>
</pre></div>
</div>
<div class="topic">
<p class="topic-title">Example:</p>
<ul class="simple">
<li><p>See <a class="reference internal" href="../auto_examples/calibration/plot_calibration.html#sphx-glr-auto-examples-calibration-plot-calibration-py"><span class="std std-ref">Probability calibration of classifiers</span></a>
for an example of Brier score loss usage to perform probability
calibration of classifiers.</p></li>
</ul>
</div>
<div class="topic">
<p class="topic-title">References:</p>
<ul class="simple">
<li><p>G. Brier, <a class="reference external" href="ftp://ftp.library.noaa.gov/docs.lib/htdocs/rescue/mwr/078/mwr-078-01-0001.pdf">Verification of forecasts expressed in terms of probability</a>,
Monthly weather review 78.1 (1950)</p></li>
</ul>
</div>
</div>
</div>
<div class="section" id="multilabel-ranking-metrics">
<span id="id31"></span><h2>3.3.3. Multilabel ranking metrics<a class="headerlink" href="#multilabel-ranking-metrics" title="Permalink to this headline">¶</a></h2>
<p>In multilabel learning, each sample can have any number of ground truth labels
associated with it. The goal is to give high scores and better rank to
the ground truth labels.</p>
<div class="section" id="coverage-error">
<span id="id32"></span><h3>3.3.3.1. Coverage error<a class="headerlink" href="#coverage-error" title="Permalink to this headline">¶</a></h3>
<p>The <a class="reference internal" href="generated/sklearn.metrics.coverage_error.html#sklearn.metrics.coverage_error" title="sklearn.metrics.coverage_error"><code class="xref py py-func docutils literal notranslate"><span class="pre">coverage_error</span></code></a> function computes the average number of labels that
have to be included in the final prediction such that all true labels
are predicted. This is useful if you want to know how many top-scored-labels
you have to predict in average without missing any true one. The best value
of this metrics is thus the average number of true labels.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>Our implementation’s score is 1 greater than the one given in Tsoumakas
et al., 2010. This extends it to handle the degenerate case in which an
instance has 0 true labels.</p>
</div>
<p>Formally, given a binary indicator matrix of the ground truth labels
<span class="math notranslate nohighlight">\(y \in \left\{0, 1\right\}^{n_\text{samples} \times n_\text{labels}}\)</span> and the
score associated with each label
<span class="math notranslate nohighlight">\(\hat{f} \in \mathbb{R}^{n_\text{samples} \times n_\text{labels}}\)</span>,
the coverage is defined as</p>
<div class="math notranslate nohighlight">
\[coverage(y, \hat{f}) = \frac{1}{n_{\text{samples}}}
  \sum_{i=0}^{n_{\text{samples}} - 1} \max_{j:y_{ij} = 1} \text{rank}_{ij}\]</div>
<p>with <span class="math notranslate nohighlight">\(\text{rank}_{ij} = \left|\left\{k: \hat{f}_{ik} \geq \hat{f}_{ij} \right\}\right|\)</span>.
Given the rank definition, ties in <code class="docutils literal notranslate"><span class="pre">y_scores</span></code> are broken by giving the
maximal rank that would have been assigned to all tied values.</p>
<p>Here is a small example of usage of this function:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="kn">import</span> <span class="n">coverage_error</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_true</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">],</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_score</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([[</span><span class="mf">0.75</span><span class="p">,</span> <span class="mf">0.5</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mf">0.2</span><span class="p">,</span> <span class="mf">0.1</span><span class="p">]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">coverage_error</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_score</span><span class="p">)</span>
<span class="go">2.5</span>
</pre></div>
</div>
</div>
<div class="section" id="label-ranking-average-precision">
<span id="id33"></span><h3>3.3.3.2. Label ranking average precision<a class="headerlink" href="#label-ranking-average-precision" title="Permalink to this headline">¶</a></h3>
<p>The <a class="reference internal" href="generated/sklearn.metrics.label_ranking_average_precision_score.html#sklearn.metrics.label_ranking_average_precision_score" title="sklearn.metrics.label_ranking_average_precision_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">label_ranking_average_precision_score</span></code></a> function
implements label ranking average precision (LRAP). This metric is linked to
the <a class="reference internal" href="generated/sklearn.metrics.average_precision_score.html#sklearn.metrics.average_precision_score" title="sklearn.metrics.average_precision_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">average_precision_score</span></code></a> function, but is based on the notion of
label ranking instead of precision and recall.</p>
<p>Label ranking average precision (LRAP) averages over the samples the answer to
the following question: for each ground truth label, what fraction of
higher-ranked labels were true labels? This performance measure will be higher
if you are able to give better rank to the labels associated with each sample.
The obtained score is always strictly greater than 0, and the best value is 1.
If there is exactly one relevant label per sample, label ranking average
precision is equivalent to the <a class="reference external" href="https://en.wikipedia.org/wiki/Mean_reciprocal_rank">mean
reciprocal rank</a>.</p>
<p>Formally, given a binary indicator matrix of the ground truth labels
<span class="math notranslate nohighlight">\(y \in \left\{0, 1\right\}^{n_\text{samples} \times n_\text{labels}}\)</span>
and the score associated with each label
<span class="math notranslate nohighlight">\(\hat{f} \in \mathbb{R}^{n_\text{samples} \times n_\text{labels}}\)</span>,
the average precision is defined as</p>
<div class="math notranslate nohighlight">
\[LRAP(y, \hat{f}) = \frac{1}{n_{\text{samples}}}
  \sum_{i=0}^{n_{\text{samples}} - 1} \frac{1}{||y_i||_0}
  \sum_{j:y_{ij} = 1} \frac{|\mathcal{L}_{ij}|}{\text{rank}_{ij}}\]</div>
<p>where
<span class="math notranslate nohighlight">\(\mathcal{L}_{ij} = \left\{k: y_{ik} = 1, \hat{f}_{ik} \geq \hat{f}_{ij} \right\}\)</span>,
<span class="math notranslate nohighlight">\(\text{rank}_{ij} = \left|\left\{k: \hat{f}_{ik} \geq \hat{f}_{ij} \right\}\right|\)</span>,
<span class="math notranslate nohighlight">\(|\cdot|\)</span> computes the cardinality of the set (i.e., the number of
elements in the set), and <span class="math notranslate nohighlight">\(||\cdot||_0\)</span> is the <span class="math notranslate nohighlight">\(\ell_0\)</span> “norm”
(which computes the number of nonzero elements in a vector).</p>
<p>Here is a small example of usage of this function:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="kn">import</span> <span class="n">label_ranking_average_precision_score</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_true</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">],</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_score</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([[</span><span class="mf">0.75</span><span class="p">,</span> <span class="mf">0.5</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mf">0.2</span><span class="p">,</span> <span class="mf">0.1</span><span class="p">]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">label_ranking_average_precision_score</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_score</span><span class="p">)</span>
<span class="go">0.416...</span>
</pre></div>
</div>
</div>
<div class="section" id="ranking-loss">
<span id="label-ranking-loss"></span><h3>3.3.3.3. Ranking loss<a class="headerlink" href="#ranking-loss" title="Permalink to this headline">¶</a></h3>
<p>The <a class="reference internal" href="generated/sklearn.metrics.label_ranking_loss.html#sklearn.metrics.label_ranking_loss" title="sklearn.metrics.label_ranking_loss"><code class="xref py py-func docutils literal notranslate"><span class="pre">label_ranking_loss</span></code></a> function computes the ranking loss which
averages over the samples the number of label pairs that are incorrectly
ordered, i.e. true labels have a lower score than false labels, weighted by
the inverse of the number of ordered pairs of false and true labels.
The lowest achievable ranking loss is zero.</p>
<p>Formally, given a binary indicator matrix of the ground truth labels
<span class="math notranslate nohighlight">\(y \in \left\{0, 1\right\}^{n_\text{samples} \times n_\text{labels}}\)</span> and the
score associated with each label
<span class="math notranslate nohighlight">\(\hat{f} \in \mathbb{R}^{n_\text{samples} \times n_\text{labels}}\)</span>,
the ranking loss is defined as</p>
<div class="math notranslate nohighlight">
\[ranking\_loss(y, \hat{f}) =  \frac{1}{n_{\text{samples}}}
  \sum_{i=0}^{n_{\text{samples}} - 1} \frac{1}{||y_i||_0(n_\text{labels} - ||y_i||_0)}
  \left|\left\{(k, l): \hat{f}_{ik} \leq \hat{f}_{il}, y_{ik} = 1, y_{il} = 0 \right\}\right|\]</div>
<p>where <span class="math notranslate nohighlight">\(|\cdot|\)</span> computes the cardinality of the set (i.e., the number of
elements in the set) and <span class="math notranslate nohighlight">\(||\cdot||_0\)</span> is the <span class="math notranslate nohighlight">\(\ell_0\)</span> “norm”
(which computes the number of nonzero elements in a vector).</p>
<p>Here is a small example of usage of this function:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="kn">import</span> <span class="n">label_ranking_loss</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_true</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">],</span> <span class="p">[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">0</span><span class="p">,</span> <span class="mi">1</span><span class="p">]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_score</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([[</span><span class="mf">0.75</span><span class="p">,</span> <span class="mf">0.5</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mf">0.2</span><span class="p">,</span> <span class="mf">0.1</span><span class="p">]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">label_ranking_loss</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_score</span><span class="p">)</span>
<span class="go">0.75...</span>
<span class="gp">&gt;&gt;&gt; </span><span class="c1"># With the following prediction, we have perfect and minimal loss</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_score</span> <span class="o">=</span> <span class="n">np</span><span class="o">.</span><span class="n">array</span><span class="p">([[</span><span class="mf">1.0</span><span class="p">,</span> <span class="mf">0.1</span><span class="p">,</span> <span class="mf">0.2</span><span class="p">],</span> <span class="p">[</span><span class="mf">0.1</span><span class="p">,</span> <span class="mf">0.2</span><span class="p">,</span> <span class="mf">0.9</span><span class="p">]])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">label_ranking_loss</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_score</span><span class="p">)</span>
<span class="go">0.0</span>
</pre></div>
</div>
<div class="topic">
<p class="topic-title">References:</p>
<ul class="simple">
<li><p>Tsoumakas, G., Katakis, I., &amp; Vlahavas, I. (2010). Mining multi-label data. In
Data mining and knowledge discovery handbook (pp. 667-685). Springer US.</p></li>
</ul>
</div>
</div>
<div class="section" id="normalized-discounted-cumulative-gain">
<span id="ndcg"></span><h3>3.3.3.4. Normalized Discounted Cumulative Gain<a class="headerlink" href="#normalized-discounted-cumulative-gain" title="Permalink to this headline">¶</a></h3>
<p>Discounted Cumulative Gain (DCG) and Normalized Discounted Cumulative Gain
(NDCG) are ranking metrics; they compare a predicted order to ground-truth
scores, such as the relevance of answers to a query.</p>
<p>from the Wikipedia page for Discounted Cumulative Gain:</p>
<p>“Discounted cumulative gain (DCG) is a measure of ranking quality. In
information retrieval, it is often used to measure effectiveness of web search
engine algorithms or related applications. Using a graded relevance scale of
documents in a search-engine result set, DCG measures the usefulness, or gain,
of a document based on its position in the result list. The gain is accumulated
from the top of the result list to the bottom, with the gain of each result
discounted at lower ranks”</p>
<p>DCG orders the true targets (e.g. relevance of query answers) in the predicted
order, then multiplies them by a logarithmic decay and sums the result. The sum
can be truncated after the first <span class="math notranslate nohighlight">\(K\)</span> results, in which case we call it
DCG&#64;K.
NDCG, or NDCG&#64;K is DCG divided by the DCG obtained by a perfect prediction, so
that it is always between 0 and 1. Usually, NDCG is preferred to DCG.</p>
<p>Compared with the ranking loss, NDCG can take into account relevance scores,
rather than a ground-truth ranking. So if the ground-truth consists only of an
ordering, the ranking loss should be preferred; if the ground-truth consists of
actual usefulness scores (e.g. 0 for irrelevant, 1 for relevant, 2 for very
relevant), NDCG can be used.</p>
<p>For one sample, given the vector of continuous ground-truth values for each
target <span class="math notranslate nohighlight">\(y \in \mathbb{R}^{M}\)</span>, where <span class="math notranslate nohighlight">\(M\)</span> is the number of outputs, and
the prediction <span class="math notranslate nohighlight">\(\hat{y}\)</span>, which induces the ranking function <span class="math notranslate nohighlight">\(f\)</span>, the
DCG score is</p>
<div class="math notranslate nohighlight">
\[\sum_{r=1}^{\min(K, M)}\frac{y_{f(r)}}{\log(1 + r)}\]</div>
<p>and the NDCG score is the DCG score divided by the DCG score obtained for
<span class="math notranslate nohighlight">\(y\)</span>.</p>
<div class="topic">
<p class="topic-title">References:</p>
<ul class="simple">
<li><p>Wikipedia entry for Discounted Cumulative Gain:
<a class="reference external" href="https://en.wikipedia.org/wiki/Discounted_cumulative_gain">https://en.wikipedia.org/wiki/Discounted_cumulative_gain</a></p></li>
<li><p>Jarvelin, K., &amp; Kekalainen, J. (2002).
Cumulated gain-based evaluation of IR techniques. ACM Transactions on
Information Systems (TOIS), 20(4), 422-446.</p></li>
<li><p>Wang, Y., Wang, L., Li, Y., He, D., Chen, W., &amp; Liu, T. Y. (2013, May).
A theoretical analysis of NDCG ranking measures. In Proceedings of the 26th
Annual Conference on Learning Theory (COLT 2013)</p></li>
<li><p>McSherry, F., &amp; Najork, M. (2008, March). Computing information retrieval
performance measures efficiently in the presence of tied scores. In
European conference on information retrieval (pp. 414-421). Springer,
Berlin, Heidelberg.</p></li>
</ul>
</div>
</div>
</div>
<div class="section" id="regression-metrics">
<span id="id34"></span><h2>3.3.4. Regression metrics<a class="headerlink" href="#regression-metrics" title="Permalink to this headline">¶</a></h2>
<p>The <a class="reference internal" href="classes.html#module-sklearn.metrics" title="sklearn.metrics"><code class="xref py py-mod docutils literal notranslate"><span class="pre">sklearn.metrics</span></code></a> module implements several loss, score, and utility
functions to measure regression performance. Some of those have been enhanced
to handle the multioutput case: <a class="reference internal" href="generated/sklearn.metrics.mean_squared_error.html#sklearn.metrics.mean_squared_error" title="sklearn.metrics.mean_squared_error"><code class="xref py py-func docutils literal notranslate"><span class="pre">mean_squared_error</span></code></a>,
<a class="reference internal" href="generated/sklearn.metrics.mean_absolute_error.html#sklearn.metrics.mean_absolute_error" title="sklearn.metrics.mean_absolute_error"><code class="xref py py-func docutils literal notranslate"><span class="pre">mean_absolute_error</span></code></a>, <a class="reference internal" href="generated/sklearn.metrics.explained_variance_score.html#sklearn.metrics.explained_variance_score" title="sklearn.metrics.explained_variance_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">explained_variance_score</span></code></a> and
<a class="reference internal" href="generated/sklearn.metrics.r2_score.html#sklearn.metrics.r2_score" title="sklearn.metrics.r2_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">r2_score</span></code></a>.</p>
<p>These functions have an <code class="docutils literal notranslate"><span class="pre">multioutput</span></code> keyword argument which specifies the
way the scores or losses for each individual target should be averaged. The
default is <code class="docutils literal notranslate"><span class="pre">'uniform_average'</span></code>, which specifies a uniformly weighted mean
over outputs. If an <code class="docutils literal notranslate"><span class="pre">ndarray</span></code> of shape <code class="docutils literal notranslate"><span class="pre">(n_outputs,)</span></code> is passed, then its
entries are interpreted as weights and an according weighted average is
returned. If <code class="docutils literal notranslate"><span class="pre">multioutput</span></code> is <code class="docutils literal notranslate"><span class="pre">'raw_values'</span></code> is specified, then all
unaltered individual scores or losses will be returned in an array of shape
<code class="docutils literal notranslate"><span class="pre">(n_outputs,)</span></code>.</p>
<p>The <a class="reference internal" href="generated/sklearn.metrics.r2_score.html#sklearn.metrics.r2_score" title="sklearn.metrics.r2_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">r2_score</span></code></a> and <a class="reference internal" href="generated/sklearn.metrics.explained_variance_score.html#sklearn.metrics.explained_variance_score" title="sklearn.metrics.explained_variance_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">explained_variance_score</span></code></a> accept an additional
value <code class="docutils literal notranslate"><span class="pre">'variance_weighted'</span></code> for the <code class="docutils literal notranslate"><span class="pre">multioutput</span></code> parameter. This option
leads to a weighting of each individual score by the variance of the
corresponding target variable. This setting quantifies the globally captured
unscaled variance. If the target variables are of different scale, then this
score puts more importance on well explaining the higher variance variables.
<code class="docutils literal notranslate"><span class="pre">multioutput='variance_weighted'</span></code> is the default value for <a class="reference internal" href="generated/sklearn.metrics.r2_score.html#sklearn.metrics.r2_score" title="sklearn.metrics.r2_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">r2_score</span></code></a>
for backward compatibility. This will be changed to <code class="docutils literal notranslate"><span class="pre">uniform_average</span></code> in the
future.</p>
<div class="section" id="explained-variance-score">
<span id="id35"></span><h3>3.3.4.1. Explained variance score<a class="headerlink" href="#explained-variance-score" title="Permalink to this headline">¶</a></h3>
<p>The <a class="reference internal" href="generated/sklearn.metrics.explained_variance_score.html#sklearn.metrics.explained_variance_score" title="sklearn.metrics.explained_variance_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">explained_variance_score</span></code></a> computes the <a class="reference external" href="https://en.wikipedia.org/wiki/Explained_variation">explained variance
regression score</a>.</p>
<p>If <span class="math notranslate nohighlight">\(\hat{y}\)</span> is the estimated target output, <span class="math notranslate nohighlight">\(y\)</span> the corresponding
(correct) target output, and <span class="math notranslate nohighlight">\(Var\)</span> is <a class="reference external" href="https://en.wikipedia.org/wiki/Variance">Variance</a>, the square of the standard deviation,
then the explained variance is estimated as follow:</p>
<div class="math notranslate nohighlight">
\[explained\_{}variance(y, \hat{y}) = 1 - \frac{Var\{ y - \hat{y}\}}{Var\{y\}}\]</div>
<p>The best possible score is 1.0, lower values are worse.</p>
<p>Here is a small example of usage of the <a class="reference internal" href="generated/sklearn.metrics.explained_variance_score.html#sklearn.metrics.explained_variance_score" title="sklearn.metrics.explained_variance_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">explained_variance_score</span></code></a>
function:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="kn">import</span> <span class="n">explained_variance_score</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_true</span> <span class="o">=</span> <span class="p">[</span><span class="mi">3</span><span class="p">,</span> <span class="o">-</span><span class="mf">0.5</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">7</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_pred</span> <span class="o">=</span> <span class="p">[</span><span class="mf">2.5</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">8</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">explained_variance_score</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">)</span>
<span class="go">0.957...</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_true</span> <span class="o">=</span> <span class="p">[[</span><span class="mf">0.5</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span> <span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span> <span class="p">[</span><span class="mi">7</span><span class="p">,</span> <span class="o">-</span><span class="mi">6</span><span class="p">]]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_pred</span> <span class="o">=</span> <span class="p">[[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="p">[</span><span class="mi">8</span><span class="p">,</span> <span class="o">-</span><span class="mi">5</span><span class="p">]]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">explained_variance_score</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">,</span> <span class="n">multioutput</span><span class="o">=</span><span class="s1">&#39;raw_values&#39;</span><span class="p">)</span>
<span class="go">array([0.967..., 1.        ])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">explained_variance_score</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">,</span> <span class="n">multioutput</span><span class="o">=</span><span class="p">[</span><span class="mf">0.3</span><span class="p">,</span> <span class="mf">0.7</span><span class="p">])</span>
<span class="go">0.990...</span>
</pre></div>
</div>
</div>
<div class="section" id="max-error">
<span id="id36"></span><h3>3.3.4.2. Max error<a class="headerlink" href="#max-error" title="Permalink to this headline">¶</a></h3>
<p>The <a class="reference internal" href="generated/sklearn.metrics.max_error.html#sklearn.metrics.max_error" title="sklearn.metrics.max_error"><code class="xref py py-func docutils literal notranslate"><span class="pre">max_error</span></code></a> function computes the maximum <a class="reference external" href="https://en.wikipedia.org/wiki/Errors_and_residuals">residual error</a> , a metric
that captures the worst case error between the predicted value and
the true value. In a perfectly fitted single output regression
model, <code class="docutils literal notranslate"><span class="pre">max_error</span></code> would be <code class="docutils literal notranslate"><span class="pre">0</span></code> on the training set and though this
would be highly unlikely in the real world, this metric shows the
extent of error that the model had when it was fitted.</p>
<p>If <span class="math notranslate nohighlight">\(\hat{y}_i\)</span> is the predicted value of the <span class="math notranslate nohighlight">\(i\)</span>-th sample,
and <span class="math notranslate nohighlight">\(y_i\)</span> is the corresponding true value, then the max error is
defined as</p>
<div class="math notranslate nohighlight">
\[\text{Max Error}(y, \hat{y}) = max(| y_i - \hat{y}_i |)\]</div>
<p>Here is a small example of usage of the <a class="reference internal" href="generated/sklearn.metrics.max_error.html#sklearn.metrics.max_error" title="sklearn.metrics.max_error"><code class="xref py py-func docutils literal notranslate"><span class="pre">max_error</span></code></a> function:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="kn">import</span> <span class="n">max_error</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_true</span> <span class="o">=</span> <span class="p">[</span><span class="mi">3</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">7</span><span class="p">,</span> <span class="mi">1</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_pred</span> <span class="o">=</span> <span class="p">[</span><span class="mi">9</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">7</span><span class="p">,</span> <span class="mi">1</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">max_error</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">)</span>
<span class="go">6</span>
</pre></div>
</div>
<p>The <a class="reference internal" href="generated/sklearn.metrics.max_error.html#sklearn.metrics.max_error" title="sklearn.metrics.max_error"><code class="xref py py-func docutils literal notranslate"><span class="pre">max_error</span></code></a> does not support multioutput.</p>
</div>
<div class="section" id="mean-absolute-error">
<span id="id37"></span><h3>3.3.4.3. Mean absolute error<a class="headerlink" href="#mean-absolute-error" title="Permalink to this headline">¶</a></h3>
<p>The <a class="reference internal" href="generated/sklearn.metrics.mean_absolute_error.html#sklearn.metrics.mean_absolute_error" title="sklearn.metrics.mean_absolute_error"><code class="xref py py-func docutils literal notranslate"><span class="pre">mean_absolute_error</span></code></a> function computes <a class="reference external" href="https://en.wikipedia.org/wiki/Mean_absolute_error">mean absolute
error</a>, a risk
metric corresponding to the expected value of the absolute error loss or
<span class="math notranslate nohighlight">\(l1\)</span>-norm loss.</p>
<p>If <span class="math notranslate nohighlight">\(\hat{y}_i\)</span> is the predicted value of the <span class="math notranslate nohighlight">\(i\)</span>-th sample,
and <span class="math notranslate nohighlight">\(y_i\)</span> is the corresponding true value, then the mean absolute error
(MAE) estimated over <span class="math notranslate nohighlight">\(n_{\text{samples}}\)</span> is defined as</p>
<div class="math notranslate nohighlight">
\[\text{MAE}(y, \hat{y}) = \frac{1}{n_{\text{samples}}} \sum_{i=0}^{n_{\text{samples}}-1} \left| y_i - \hat{y}_i \right|.\]</div>
<p>Here is a small example of usage of the <a class="reference internal" href="generated/sklearn.metrics.mean_absolute_error.html#sklearn.metrics.mean_absolute_error" title="sklearn.metrics.mean_absolute_error"><code class="xref py py-func docutils literal notranslate"><span class="pre">mean_absolute_error</span></code></a> function:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="kn">import</span> <span class="n">mean_absolute_error</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_true</span> <span class="o">=</span> <span class="p">[</span><span class="mi">3</span><span class="p">,</span> <span class="o">-</span><span class="mf">0.5</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">7</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_pred</span> <span class="o">=</span> <span class="p">[</span><span class="mf">2.5</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">8</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">mean_absolute_error</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">)</span>
<span class="go">0.5</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_true</span> <span class="o">=</span> <span class="p">[[</span><span class="mf">0.5</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span> <span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span> <span class="p">[</span><span class="mi">7</span><span class="p">,</span> <span class="o">-</span><span class="mi">6</span><span class="p">]]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_pred</span> <span class="o">=</span> <span class="p">[[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="p">[</span><span class="mi">8</span><span class="p">,</span> <span class="o">-</span><span class="mi">5</span><span class="p">]]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">mean_absolute_error</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">)</span>
<span class="go">0.75</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">mean_absolute_error</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">,</span> <span class="n">multioutput</span><span class="o">=</span><span class="s1">&#39;raw_values&#39;</span><span class="p">)</span>
<span class="go">array([0.5, 1. ])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">mean_absolute_error</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">,</span> <span class="n">multioutput</span><span class="o">=</span><span class="p">[</span><span class="mf">0.3</span><span class="p">,</span> <span class="mf">0.7</span><span class="p">])</span>
<span class="go">0.85...</span>
</pre></div>
</div>
</div>
<div class="section" id="mean-squared-error">
<span id="id39"></span><h3>3.3.4.4. Mean squared error<a class="headerlink" href="#mean-squared-error" title="Permalink to this headline">¶</a></h3>
<p>The <a class="reference internal" href="generated/sklearn.metrics.mean_squared_error.html#sklearn.metrics.mean_squared_error" title="sklearn.metrics.mean_squared_error"><code class="xref py py-func docutils literal notranslate"><span class="pre">mean_squared_error</span></code></a> function computes <a class="reference external" href="https://en.wikipedia.org/wiki/Mean_squared_error">mean square
error</a>, a risk
metric corresponding to the expected value of the squared (quadratic) error or
loss.</p>
<p>If <span class="math notranslate nohighlight">\(\hat{y}_i\)</span> is the predicted value of the <span class="math notranslate nohighlight">\(i\)</span>-th sample,
and <span class="math notranslate nohighlight">\(y_i\)</span> is the corresponding true value, then the mean squared error
(MSE) estimated over <span class="math notranslate nohighlight">\(n_{\text{samples}}\)</span> is defined as</p>
<div class="math notranslate nohighlight">
\[\text{MSE}(y, \hat{y}) = \frac{1}{n_\text{samples}} \sum_{i=0}^{n_\text{samples} - 1} (y_i - \hat{y}_i)^2.\]</div>
<p>Here is a small example of usage of the <a class="reference internal" href="generated/sklearn.metrics.mean_squared_error.html#sklearn.metrics.mean_squared_error" title="sklearn.metrics.mean_squared_error"><code class="xref py py-func docutils literal notranslate"><span class="pre">mean_squared_error</span></code></a>
function:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="kn">import</span> <span class="n">mean_squared_error</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_true</span> <span class="o">=</span> <span class="p">[</span><span class="mi">3</span><span class="p">,</span> <span class="o">-</span><span class="mf">0.5</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">7</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_pred</span> <span class="o">=</span> <span class="p">[</span><span class="mf">2.5</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">8</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">mean_squared_error</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">)</span>
<span class="go">0.375</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_true</span> <span class="o">=</span> <span class="p">[[</span><span class="mf">0.5</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span> <span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span> <span class="p">[</span><span class="mi">7</span><span class="p">,</span> <span class="o">-</span><span class="mi">6</span><span class="p">]]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_pred</span> <span class="o">=</span> <span class="p">[[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="p">[</span><span class="mi">8</span><span class="p">,</span> <span class="o">-</span><span class="mi">5</span><span class="p">]]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">mean_squared_error</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">)</span>
<span class="go">0.7083...</span>
</pre></div>
</div>
<div class="topic">
<p class="topic-title">Examples:</p>
<ul class="simple">
<li><p>See <a class="reference internal" href="../auto_examples/ensemble/plot_gradient_boosting_regression.html#sphx-glr-auto-examples-ensemble-plot-gradient-boosting-regression-py"><span class="std std-ref">Gradient Boosting regression</span></a>
for an example of mean squared error usage to
evaluate gradient boosting regression.</p></li>
</ul>
</div>
</div>
<div class="section" id="mean-squared-logarithmic-error">
<span id="mean-squared-log-error"></span><h3>3.3.4.5. Mean squared logarithmic error<a class="headerlink" href="#mean-squared-logarithmic-error" title="Permalink to this headline">¶</a></h3>
<p>The <a class="reference internal" href="generated/sklearn.metrics.mean_squared_log_error.html#sklearn.metrics.mean_squared_log_error" title="sklearn.metrics.mean_squared_log_error"><code class="xref py py-func docutils literal notranslate"><span class="pre">mean_squared_log_error</span></code></a> function computes a risk metric
corresponding to the expected value of the squared logarithmic (quadratic)
error or loss.</p>
<p>If <span class="math notranslate nohighlight">\(\hat{y}_i\)</span> is the predicted value of the <span class="math notranslate nohighlight">\(i\)</span>-th sample,
and <span class="math notranslate nohighlight">\(y_i\)</span> is the corresponding true value, then the mean squared
logarithmic error (MSLE) estimated over <span class="math notranslate nohighlight">\(n_{\text{samples}}\)</span> is
defined as</p>
<div class="math notranslate nohighlight">
\[\text{MSLE}(y, \hat{y}) = \frac{1}{n_\text{samples}} \sum_{i=0}^{n_\text{samples} - 1} (\log_e (1 + y_i) - \log_e (1 + \hat{y}_i) )^2.\]</div>
<p>Where <span class="math notranslate nohighlight">\(\log_e (x)\)</span> means the natural logarithm of <span class="math notranslate nohighlight">\(x\)</span>. This metric
is best to use when targets having exponential growth, such as population
counts, average sales of a commodity over a span of years etc. Note that this
metric penalizes an under-predicted estimate greater than an over-predicted
estimate.</p>
<p>Here is a small example of usage of the <a class="reference internal" href="generated/sklearn.metrics.mean_squared_log_error.html#sklearn.metrics.mean_squared_log_error" title="sklearn.metrics.mean_squared_log_error"><code class="xref py py-func docutils literal notranslate"><span class="pre">mean_squared_log_error</span></code></a>
function:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="kn">import</span> <span class="n">mean_squared_log_error</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_true</span> <span class="o">=</span> <span class="p">[</span><span class="mi">3</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mf">2.5</span><span class="p">,</span> <span class="mi">7</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_pred</span> <span class="o">=</span> <span class="p">[</span><span class="mf">2.5</span><span class="p">,</span> <span class="mi">5</span><span class="p">,</span> <span class="mi">4</span><span class="p">,</span> <span class="mi">8</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">mean_squared_log_error</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">)</span>
<span class="go">0.039...</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_true</span> <span class="o">=</span> <span class="p">[[</span><span class="mf">0.5</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="p">[</span><span class="mi">7</span><span class="p">,</span> <span class="mi">6</span><span class="p">]]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_pred</span> <span class="o">=</span> <span class="p">[[</span><span class="mf">0.5</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mf">2.5</span><span class="p">],</span> <span class="p">[</span><span class="mi">8</span><span class="p">,</span> <span class="mi">8</span><span class="p">]]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">mean_squared_log_error</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">)</span>
<span class="go">0.044...</span>
</pre></div>
</div>
</div>
<div class="section" id="median-absolute-error">
<span id="id40"></span><h3>3.3.4.6. Median absolute error<a class="headerlink" href="#median-absolute-error" title="Permalink to this headline">¶</a></h3>
<p>The <a class="reference internal" href="generated/sklearn.metrics.median_absolute_error.html#sklearn.metrics.median_absolute_error" title="sklearn.metrics.median_absolute_error"><code class="xref py py-func docutils literal notranslate"><span class="pre">median_absolute_error</span></code></a> is particularly interesting because it is
robust to outliers. The loss is calculated by taking the median of all absolute
differences between the target and the prediction.</p>
<p>If <span class="math notranslate nohighlight">\(\hat{y}_i\)</span> is the predicted value of the <span class="math notranslate nohighlight">\(i\)</span>-th sample
and <span class="math notranslate nohighlight">\(y_i\)</span> is the corresponding true value, then the median absolute error
(MedAE) estimated over <span class="math notranslate nohighlight">\(n_{\text{samples}}\)</span> is defined as</p>
<div class="math notranslate nohighlight">
\[\text{MedAE}(y, \hat{y}) = \text{median}(\mid y_1 - \hat{y}_1 \mid, \ldots, \mid y_n - \hat{y}_n \mid).\]</div>
<p>The <a class="reference internal" href="generated/sklearn.metrics.median_absolute_error.html#sklearn.metrics.median_absolute_error" title="sklearn.metrics.median_absolute_error"><code class="xref py py-func docutils literal notranslate"><span class="pre">median_absolute_error</span></code></a> does not support multioutput.</p>
<p>Here is a small example of usage of the <a class="reference internal" href="generated/sklearn.metrics.median_absolute_error.html#sklearn.metrics.median_absolute_error" title="sklearn.metrics.median_absolute_error"><code class="xref py py-func docutils literal notranslate"><span class="pre">median_absolute_error</span></code></a>
function:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="kn">import</span> <span class="n">median_absolute_error</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_true</span> <span class="o">=</span> <span class="p">[</span><span class="mi">3</span><span class="p">,</span> <span class="o">-</span><span class="mf">0.5</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">7</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_pred</span> <span class="o">=</span> <span class="p">[</span><span class="mf">2.5</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">8</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">median_absolute_error</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">)</span>
<span class="go">0.5</span>
</pre></div>
</div>
</div>
<div class="section" id="r2-score-the-coefficient-of-determination">
<span id="r2-score"></span><h3>3.3.4.7. R² score, the coefficient of determination<a class="headerlink" href="#r2-score-the-coefficient-of-determination" title="Permalink to this headline">¶</a></h3>
<p>The <a class="reference internal" href="generated/sklearn.metrics.r2_score.html#sklearn.metrics.r2_score" title="sklearn.metrics.r2_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">r2_score</span></code></a> function computes the <a class="reference external" href="https://en.wikipedia.org/wiki/Coefficient_of_determination">coefficient of
determination</a>,
usually denoted as R².</p>
<p>It represents the proportion of variance (of y) that has been explained by the
independent variables in the model. It provides an indication of goodness of
fit and therefore a measure of how well unseen samples are likely to be
predicted by the model, through the proportion of explained variance.</p>
<p>As such variance is dataset dependent, R² may not be meaningfully comparable
across different datasets. Best possible score is 1.0 and it can be negative
(because the model can be arbitrarily worse). A constant model that always
predicts the expected value of y, disregarding the input features, would get a
R² score of 0.0.</p>
<p>If <span class="math notranslate nohighlight">\(\hat{y}_i\)</span> is the predicted value of the <span class="math notranslate nohighlight">\(i\)</span>-th sample
and <span class="math notranslate nohighlight">\(y_i\)</span> is the corresponding true value for total <span class="math notranslate nohighlight">\(n\)</span> samples,
the estimated R² is defined as:</p>
<div class="math notranslate nohighlight">
\[R^2(y, \hat{y}) = 1 - \frac{\sum_{i=1}^{n} (y_i - \hat{y}_i)^2}{\sum_{i=1}^{n} (y_i - \bar{y})^2}\]</div>
<p>where <span class="math notranslate nohighlight">\(\bar{y} = \frac{1}{n} \sum_{i=1}^{n} y_i\)</span> and <span class="math notranslate nohighlight">\(\sum_{i=1}^{n} (y_i - \hat{y}_i)^2 = \sum_{i=1}^{n} \epsilon_i^2\)</span>.</p>
<p>Note that <a class="reference internal" href="generated/sklearn.metrics.r2_score.html#sklearn.metrics.r2_score" title="sklearn.metrics.r2_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">r2_score</span></code></a> calculates unadjusted R² without correcting for
bias in sample variance of y.</p>
<p>Here is a small example of usage of the <a class="reference internal" href="generated/sklearn.metrics.r2_score.html#sklearn.metrics.r2_score" title="sklearn.metrics.r2_score"><code class="xref py py-func docutils literal notranslate"><span class="pre">r2_score</span></code></a> function:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="kn">import</span> <span class="n">r2_score</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_true</span> <span class="o">=</span> <span class="p">[</span><span class="mi">3</span><span class="p">,</span> <span class="o">-</span><span class="mf">0.5</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">7</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_pred</span> <span class="o">=</span> <span class="p">[</span><span class="mf">2.5</span><span class="p">,</span> <span class="mf">0.0</span><span class="p">,</span> <span class="mi">2</span><span class="p">,</span> <span class="mi">8</span><span class="p">]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">r2_score</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">)</span>
<span class="go">0.948...</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_true</span> <span class="o">=</span> <span class="p">[[</span><span class="mf">0.5</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span> <span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span> <span class="p">[</span><span class="mi">7</span><span class="p">,</span> <span class="o">-</span><span class="mi">6</span><span class="p">]]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_pred</span> <span class="o">=</span> <span class="p">[[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="p">[</span><span class="mi">8</span><span class="p">,</span> <span class="o">-</span><span class="mi">5</span><span class="p">]]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">r2_score</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">,</span> <span class="n">multioutput</span><span class="o">=</span><span class="s1">&#39;variance_weighted&#39;</span><span class="p">)</span>
<span class="go">0.938...</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_true</span> <span class="o">=</span> <span class="p">[[</span><span class="mf">0.5</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span> <span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">1</span><span class="p">],</span> <span class="p">[</span><span class="mi">7</span><span class="p">,</span> <span class="o">-</span><span class="mi">6</span><span class="p">]]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y_pred</span> <span class="o">=</span> <span class="p">[[</span><span class="mi">0</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span> <span class="p">[</span><span class="mi">8</span><span class="p">,</span> <span class="o">-</span><span class="mi">5</span><span class="p">]]</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">r2_score</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">,</span> <span class="n">multioutput</span><span class="o">=</span><span class="s1">&#39;uniform_average&#39;</span><span class="p">)</span>
<span class="go">0.936...</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">r2_score</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">,</span> <span class="n">multioutput</span><span class="o">=</span><span class="s1">&#39;raw_values&#39;</span><span class="p">)</span>
<span class="go">array([0.965..., 0.908...])</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">r2_score</span><span class="p">(</span><span class="n">y_true</span><span class="p">,</span> <span class="n">y_pred</span><span class="p">,</span> <span class="n">multioutput</span><span class="o">=</span><span class="p">[</span><span class="mf">0.3</span><span class="p">,</span> <span class="mf">0.7</span><span class="p">])</span>
<span class="go">0.925...</span>
</pre></div>
</div>
<div class="topic">
<p class="topic-title">Example:</p>
<ul class="simple">
<li><p>See <a class="reference internal" href="../auto_examples/linear_model/plot_lasso_and_elasticnet.html#sphx-glr-auto-examples-linear-model-plot-lasso-and-elasticnet-py"><span class="std std-ref">Lasso and Elastic Net for Sparse Signals</span></a>
for an example of R² score usage to
evaluate Lasso and Elastic Net on sparse signals.</p></li>
</ul>
</div>
</div>
<div class="section" id="mean-poisson-gamma-and-tweedie-deviances">
<span id="mean-tweedie-deviance"></span><h3>3.3.4.8. Mean Poisson, Gamma, and Tweedie deviances<a class="headerlink" href="#mean-poisson-gamma-and-tweedie-deviances" title="Permalink to this headline">¶</a></h3>
<p>The <a class="reference internal" href="generated/sklearn.metrics.mean_tweedie_deviance.html#sklearn.metrics.mean_tweedie_deviance" title="sklearn.metrics.mean_tweedie_deviance"><code class="xref py py-func docutils literal notranslate"><span class="pre">mean_tweedie_deviance</span></code></a> function computes the <a class="reference external" href="https://en.wikipedia.org/wiki/Tweedie_distribution#The_Tweedie_deviance">mean Tweedie
deviance error</a>
with a <code class="docutils literal notranslate"><span class="pre">power</span></code> parameter (<span class="math notranslate nohighlight">\(p\)</span>). This is a metric that elicits
predicted expectation values of regression targets.</p>
<p>Following special cases exist,</p>
<ul class="simple">
<li><p>when <code class="docutils literal notranslate"><span class="pre">power=0</span></code> it is equivalent to <a class="reference internal" href="generated/sklearn.metrics.mean_squared_error.html#sklearn.metrics.mean_squared_error" title="sklearn.metrics.mean_squared_error"><code class="xref py py-func docutils literal notranslate"><span class="pre">mean_squared_error</span></code></a>.</p></li>
<li><p>when <code class="docutils literal notranslate"><span class="pre">power=1</span></code> it is equivalent to <a class="reference internal" href="generated/sklearn.metrics.mean_poisson_deviance.html#sklearn.metrics.mean_poisson_deviance" title="sklearn.metrics.mean_poisson_deviance"><code class="xref py py-func docutils literal notranslate"><span class="pre">mean_poisson_deviance</span></code></a>.</p></li>
<li><p>when <code class="docutils literal notranslate"><span class="pre">power=2</span></code> it is equivalent to <a class="reference internal" href="generated/sklearn.metrics.mean_gamma_deviance.html#sklearn.metrics.mean_gamma_deviance" title="sklearn.metrics.mean_gamma_deviance"><code class="xref py py-func docutils literal notranslate"><span class="pre">mean_gamma_deviance</span></code></a>.</p></li>
</ul>
<p>If <span class="math notranslate nohighlight">\(\hat{y}_i\)</span> is the predicted value of the <span class="math notranslate nohighlight">\(i\)</span>-th sample,
and <span class="math notranslate nohighlight">\(y_i\)</span> is the corresponding true value, then the mean Tweedie
deviance error (D) for power <span class="math notranslate nohighlight">\(p\)</span>, estimated over <span class="math notranslate nohighlight">\(n_{\text{samples}}\)</span>
is defined as</p>
<div class="math notranslate nohighlight">
\[\begin{split}\text{D}(y, \hat{y}) = \frac{1}{n_\text{samples}}
\sum_{i=0}^{n_\text{samples} - 1}
\begin{cases}
(y_i-\hat{y}_i)^2, &amp; \text{for }p=0\text{ (Normal)}\\
2(y_i \log(y/\hat{y}_i) + \hat{y}_i - y_i),  &amp; \text{for}p=1\text{ (Poisson)}\\
2(\log(\hat{y}_i/y_i) + y_i/\hat{y}_i - 1),  &amp; \text{for}p=2\text{ (Gamma)}\\
2\left(\frac{\max(y_i,0)^{2-p}}{(1-p)(2-p)}-
\frac{y\,\hat{y}^{1-p}_i}{1-p}+\frac{\hat{y}^{2-p}_i}{2-p}\right),
&amp; \text{otherwise}
\end{cases}\end{split}\]</div>
<p>Tweedie deviance is a homogeneous function of degree <code class="docutils literal notranslate"><span class="pre">2-power</span></code>.
Thus, Gamma distribution with <code class="docutils literal notranslate"><span class="pre">power=2</span></code> means that simultaneously scaling
<code class="docutils literal notranslate"><span class="pre">y_true</span></code> and <code class="docutils literal notranslate"><span class="pre">y_pred</span></code> has no effect on the deviance. For Poisson
distribution <code class="docutils literal notranslate"><span class="pre">power=1</span></code> the deviance scales linearly, and for Normal
distribution (<code class="docutils literal notranslate"><span class="pre">power=0</span></code>), quadratically.  In general, the higher
<code class="docutils literal notranslate"><span class="pre">power</span></code> the less weight is given to extreme deviations between true
and predicted targets.</p>
<p>For instance, let’s compare the two predictions 1.0 and 100 that are both
50% of their corresponding true value.</p>
<p>The mean squared error (<code class="docutils literal notranslate"><span class="pre">power=0</span></code>) is very sensitive to the
prediction difference of the second point,:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.metrics</span> <span class="kn">import</span> <span class="n">mean_tweedie_deviance</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">mean_tweedie_deviance</span><span class="p">([</span><span class="mf">1.0</span><span class="p">],</span> <span class="p">[</span><span class="mf">1.5</span><span class="p">],</span> <span class="n">power</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
<span class="go">0.25</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">mean_tweedie_deviance</span><span class="p">([</span><span class="mf">100.</span><span class="p">],</span> <span class="p">[</span><span class="mf">150.</span><span class="p">],</span> <span class="n">power</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
<span class="go">2500.0</span>
</pre></div>
</div>
<p>If we increase <code class="docutils literal notranslate"><span class="pre">power</span></code> to 1,:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">mean_tweedie_deviance</span><span class="p">([</span><span class="mf">1.0</span><span class="p">],</span> <span class="p">[</span><span class="mf">1.5</span><span class="p">],</span> <span class="n">power</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
<span class="go">0.18...</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">mean_tweedie_deviance</span><span class="p">([</span><span class="mf">100.</span><span class="p">],</span> <span class="p">[</span><span class="mf">150.</span><span class="p">],</span> <span class="n">power</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span>
<span class="go">18.9...</span>
</pre></div>
</div>
<p>the difference in errors decreases. Finally, by setting, <code class="docutils literal notranslate"><span class="pre">power=2</span></code>:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">mean_tweedie_deviance</span><span class="p">([</span><span class="mf">1.0</span><span class="p">],</span> <span class="p">[</span><span class="mf">1.5</span><span class="p">],</span> <span class="n">power</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
<span class="go">0.14...</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">mean_tweedie_deviance</span><span class="p">([</span><span class="mf">100.</span><span class="p">],</span> <span class="p">[</span><span class="mf">150.</span><span class="p">],</span> <span class="n">power</span><span class="o">=</span><span class="mi">2</span><span class="p">)</span>
<span class="go">0.14...</span>
</pre></div>
</div>
<p>we would get identical errors. The deviance when <code class="docutils literal notranslate"><span class="pre">power=2</span></code> is thus only
sensitive to relative errors.</p>
</div>
</div>
<div class="section" id="clustering-metrics">
<span id="id41"></span><h2>3.3.5. Clustering metrics<a class="headerlink" href="#clustering-metrics" title="Permalink to this headline">¶</a></h2>
<p>The <a class="reference internal" href="classes.html#module-sklearn.metrics" title="sklearn.metrics"><code class="xref py py-mod docutils literal notranslate"><span class="pre">sklearn.metrics</span></code></a> module implements several loss, score, and utility
functions. For more information see the <a class="reference internal" href="clustering.html#clustering-evaluation"><span class="std std-ref">Clustering performance evaluation</span></a>
section for instance clustering, and <a class="reference internal" href="biclustering.html#biclustering-evaluation"><span class="std std-ref">Biclustering evaluation</span></a> for
biclustering.</p>
</div>
<div class="section" id="dummy-estimators">
<span id="id42"></span><h2>3.3.6. Dummy estimators<a class="headerlink" href="#dummy-estimators" title="Permalink to this headline">¶</a></h2>
<p>When doing supervised learning, a simple sanity check consists of comparing
one’s estimator against simple rules of thumb. <a class="reference internal" href="generated/sklearn.dummy.DummyClassifier.html#sklearn.dummy.DummyClassifier" title="sklearn.dummy.DummyClassifier"><code class="xref py py-class docutils literal notranslate"><span class="pre">DummyClassifier</span></code></a>
implements several such simple strategies for classification:</p>
<ul class="simple">
<li><p><code class="docutils literal notranslate"><span class="pre">stratified</span></code> generates random predictions by respecting the training
set class distribution.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">most_frequent</span></code> always predicts the most frequent label in the training set.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">prior</span></code> always predicts the class that maximizes the class prior
(like <code class="docutils literal notranslate"><span class="pre">most_frequent</span></code>) and <code class="docutils literal notranslate"><span class="pre">predict_proba</span></code> returns the class prior.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">uniform</span></code> generates predictions uniformly at random.</p></li>
<li><dl class="simple">
<dt><code class="docutils literal notranslate"><span class="pre">constant</span></code> always predicts a constant label that is provided by the user.</dt><dd><p>A major motivation of this method is F1-scoring, when the positive class
is in the minority.</p>
</dd>
</dl>
</li>
</ul>
<p>Note that with all these strategies, the <code class="docutils literal notranslate"><span class="pre">predict</span></code> method completely ignores
the input data!</p>
<p>To illustrate <a class="reference internal" href="generated/sklearn.dummy.DummyClassifier.html#sklearn.dummy.DummyClassifier" title="sklearn.dummy.DummyClassifier"><code class="xref py py-class docutils literal notranslate"><span class="pre">DummyClassifier</span></code></a>, first let’s create an imbalanced
dataset:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.datasets</span> <span class="kn">import</span> <span class="n">load_iris</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.model_selection</span> <span class="kn">import</span> <span class="n">train_test_split</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">X</span><span class="p">,</span> <span class="n">y</span> <span class="o">=</span> <span class="n">load_iris</span><span class="p">(</span><span class="n">return_X_y</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">y</span><span class="p">[</span><span class="n">y</span> <span class="o">!=</span> <span class="mi">1</span><span class="p">]</span> <span class="o">=</span> <span class="o">-</span><span class="mi">1</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">X_train</span><span class="p">,</span> <span class="n">X_test</span><span class="p">,</span> <span class="n">y_train</span><span class="p">,</span> <span class="n">y_test</span> <span class="o">=</span> <span class="n">train_test_split</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">,</span> <span class="n">random_state</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
</pre></div>
</div>
<p>Next, let’s compare the accuracy of <code class="docutils literal notranslate"><span class="pre">SVC</span></code> and <code class="docutils literal notranslate"><span class="pre">most_frequent</span></code>:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.dummy</span> <span class="kn">import</span> <span class="n">DummyClassifier</span>
<span class="gp">&gt;&gt;&gt; </span><span class="kn">from</span> <span class="nn">sklearn.svm</span> <span class="kn">import</span> <span class="n">SVC</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">clf</span> <span class="o">=</span> <span class="n">SVC</span><span class="p">(</span><span class="n">kernel</span><span class="o">=</span><span class="s1">&#39;linear&#39;</span><span class="p">,</span> <span class="n">C</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X_train</span><span class="p">,</span> <span class="n">y_train</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">clf</span><span class="o">.</span><span class="n">score</span><span class="p">(</span><span class="n">X_test</span><span class="p">,</span> <span class="n">y_test</span><span class="p">)</span>
<span class="go">0.63...</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">clf</span> <span class="o">=</span> <span class="n">DummyClassifier</span><span class="p">(</span><span class="n">strategy</span><span class="o">=</span><span class="s1">&#39;most_frequent&#39;</span><span class="p">,</span> <span class="n">random_state</span><span class="o">=</span><span class="mi">0</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">clf</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X_train</span><span class="p">,</span> <span class="n">y_train</span><span class="p">)</span>
<span class="go">DummyClassifier(random_state=0, strategy=&#39;most_frequent&#39;)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">clf</span><span class="o">.</span><span class="n">score</span><span class="p">(</span><span class="n">X_test</span><span class="p">,</span> <span class="n">y_test</span><span class="p">)</span>
<span class="go">0.57...</span>
</pre></div>
</div>
<p>We see that <code class="docutils literal notranslate"><span class="pre">SVC</span></code> doesn’t do much better than a dummy classifier. Now, let’s
change the kernel:</p>
<div class="highlight-default notranslate"><div class="highlight"><pre><span></span><span class="gp">&gt;&gt;&gt; </span><span class="n">clf</span> <span class="o">=</span> <span class="n">SVC</span><span class="p">(</span><span class="n">kernel</span><span class="o">=</span><span class="s1">&#39;rbf&#39;</span><span class="p">,</span> <span class="n">C</span><span class="o">=</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="n">fit</span><span class="p">(</span><span class="n">X_train</span><span class="p">,</span> <span class="n">y_train</span><span class="p">)</span>
<span class="gp">&gt;&gt;&gt; </span><span class="n">clf</span><span class="o">.</span><span class="n">score</span><span class="p">(</span><span class="n">X_test</span><span class="p">,</span> <span class="n">y_test</span><span class="p">)</span>
<span class="go">0.94...</span>
</pre></div>
</div>
<p>We see that the accuracy was boosted to almost 100%.  A cross validation
strategy is recommended for a better estimate of the accuracy, if it
is not too CPU costly. For more information see the <a class="reference internal" href="cross_validation.html#cross-validation"><span class="std std-ref">Cross-validation: evaluating estimator performance</span></a>
section. Moreover if you want to optimize over the parameter space, it is highly
recommended to use an appropriate methodology; see the <a class="reference internal" href="grid_search.html#grid-search"><span class="std std-ref">Tuning the hyper-parameters of an estimator</span></a>
section for details.</p>
<p>More generally, when the accuracy of a classifier is too close to random, it
probably means that something went wrong: features are not helpful, a
hyperparameter is not correctly tuned, the classifier is suffering from class
imbalance, etc…</p>
<p><a class="reference internal" href="generated/sklearn.dummy.DummyRegressor.html#sklearn.dummy.DummyRegressor" title="sklearn.dummy.DummyRegressor"><code class="xref py py-class docutils literal notranslate"><span class="pre">DummyRegressor</span></code></a> also implements four simple rules of thumb for regression:</p>
<ul class="simple">
<li><p><code class="docutils literal notranslate"><span class="pre">mean</span></code> always predicts the mean of the training targets.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">median</span></code> always predicts the median of the training targets.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">quantile</span></code> always predicts a user provided quantile of the training targets.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">constant</span></code> always predicts a constant value that is provided by the user.</p></li>
</ul>
<p>In all these strategies, the <code class="docutils literal notranslate"><span class="pre">predict</span></code> method completely ignores
the input data.</p>
</div>
</div>


      </div>
    <div class="container">
      <footer class="sk-content-footer">
            &copy; 2007 - 2019, scikit-learn developers (BSD License).
          <a href="../_sources/modules/model_evaluation.rst.txt" rel="nofollow">Show this page source</a>
      </footer>
    </div>
  </div>
</div>
<script src="../_static/js/vendor/bootstrap.min.js"></script>

<script>
    window.ga=window.ga||function(){(ga.q=ga.q||[]).push(arguments)};ga.l=+new Date;
    ga('create', 'UA-22606712-2', 'auto');
    ga('set', 'anonymizeIp', true);
    ga('send', 'pageview');
</script>
<script async src='https://www.google-analytics.com/analytics.js'></script>


<script>
$(document).ready(function() {
    /* Add a [>>>] button on the top-right corner of code samples to hide
     * the >>> and ... prompts and the output and thus make the code
     * copyable. */
    var div = $('.highlight-python .highlight,' +
                '.highlight-python3 .highlight,' +
                '.highlight-pycon .highlight,' +
		'.highlight-default .highlight')
    var pre = div.find('pre');

    // get the styles from the current theme
    pre.parent().parent().css('position', 'relative');
    var hide_text = 'Hide prompts and outputs';
    var show_text = 'Show prompts and outputs';

    // create and add the button to all the code blocks that contain >>>
    div.each(function(index) {
        var jthis = $(this);
        if (jthis.find('.gp').length > 0) {
            var button = $('<span class="copybutton">&gt;&gt;&gt;</span>');
            button.attr('title', hide_text);
            button.data('hidden', 'false');
            jthis.prepend(button);
        }
        // tracebacks (.gt) contain bare text elements that need to be
        // wrapped in a span to work with .nextUntil() (see later)
        jthis.find('pre:has(.gt)').contents().filter(function() {
            return ((this.nodeType == 3) && (this.data.trim().length > 0));
        }).wrap('<span>');
    });

    // define the behavior of the button when it's clicked
    $('.copybutton').click(function(e){
        e.preventDefault();
        var button = $(this);
        if (button.data('hidden') === 'false') {
            // hide the code output
            button.parent().find('.go, .gp, .gt').hide();
            button.next('pre').find('.gt').nextUntil('.gp, .go').css('visibility', 'hidden');
            button.css('text-decoration', 'line-through');
            button.attr('title', show_text);
            button.data('hidden', 'true');
        } else {
            // show the code output
            button.parent().find('.go, .gp, .gt').show();
            button.next('pre').find('.gt').nextUntil('.gp, .go').css('visibility', 'visible');
            button.css('text-decoration', 'none');
            button.attr('title', hide_text);
            button.data('hidden', 'false');
        }
    });

	/*** Add permalink buttons next to glossary terms ***/
	$('dl.glossary > dt[id]').append(function() {
		return ('<a class="headerlink" href="#' +
			    this.getAttribute('id') +
			    '" title="Permalink to this term">¶</a>');
	});
  /*** Hide navbar when scrolling down ***/
  // Returns true when headerlink target matches hash in url
  (function() {
    hashTargetOnTop = function() {
        var hash = window.location.hash;
        if ( hash.length < 2 ) { return false; }

        var target = document.getElementById( hash.slice(1) );
        if ( target === null ) { return false; }

        var top = target.getBoundingClientRect().top;
        return (top < 2) && (top > -2);
    };

    // Hide navbar on load if hash target is on top
    var navBar = document.getElementById("navbar");
    var navBarToggler = document.getElementById("sk-navbar-toggler");
    var navBarHeightHidden = "-" + navBar.getBoundingClientRect().height + "px";
    var $window = $(window);

    hideNavBar = function() {
        navBar.style.top = navBarHeightHidden;
    };

    showNavBar = function() {
        navBar.style.top = "0";
    }

    if (hashTargetOnTop()) {
        hideNavBar()
    }

    var prevScrollpos = window.pageYOffset;
    hideOnScroll = function(lastScrollTop) {
        if (($window.width() < 768) && (navBarToggler.getAttribute("aria-expanded") === 'true')) {
            return;
        }
        if (lastScrollTop > 2 && (prevScrollpos <= lastScrollTop) || hashTargetOnTop()){
            hideNavBar()
        } else {
            showNavBar()
        }
        prevScrollpos = lastScrollTop;
    };

    /*** high preformance scroll event listener***/
    var raf = window.requestAnimationFrame ||
        window.webkitRequestAnimationFrame ||
        window.mozRequestAnimationFrame ||
        window.msRequestAnimationFrame ||
        window.oRequestAnimationFrame;
    var lastScrollTop = $window.scrollTop();

    if (raf) {
        loop();
    }

    function loop() {
        var scrollTop = $window.scrollTop();
        if (lastScrollTop === scrollTop) {
            raf(loop);
            return;
        } else {
            lastScrollTop = scrollTop;
            hideOnScroll(lastScrollTop);
            raf(loop);
        }
    }
  })();
});

</script>
    
<script id="MathJax-script" async src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-chtml.js"></script>
    
</body>
</html>